{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9mcB2J1RRuE-"
      },
      "source": [
        "# GSM8K with Langfun's Evaluation Framework\n",
        "\n",
        "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/google/langfun/blob/main/docs/notebooks/langfun_eval.ipynb)\n",
        "\n",
        "This notebook demonstrates how to build LLM benchmarks using Langfun's evaluation framework. By the end of this notebook, you will learn:\n",
        "\n",
        "- How to create a new benchmark.\n",
        "- How to execute a benchmark.\n",
        "- How to view benchmark details.\n",
        "\n",
        "We recommend opening this notebook in Colab, as GitHub's file preview does not render generated HTML views correctly."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5zoBXnPcHCHq"
      },
      "outputs": [],
      "source": [
        "!pip install langfun --pre"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jQDlfhYuHKwQ"
      },
      "outputs": [],
      "source": [
        "from typing import Any\n",
        "\n",
        "import langfun as lf\n",
        "import pandas as pd\n",
        "import pyglove as pg\n",
        "\n",
        "# Evaluation Input Functor\n",
        "# NOTE: pg.functor decorator generates `pg.Object` subclass with a `__call__`\n",
        "# method. It is commonly used for early or partial binding.\n",
        "@pg.functor()\n",
        "def gsm8k_inputs(num_examples=10):\n",
        "  splits = {'train': 'main/train-00000-of-00001.parquet', 'test': 'main/test-00000-of-00001.parquet'}\n",
        "  df = pd.read_parquet(\"hf://datasets/openai/gsm8k/\" + splits[\"test\"])\n",
        "  return [\n",
        "      pg.Dict(\n",
        "          question=df['question'][i],\n",
        "          answer=int(df['answer'][i].split('#### ')[1]),\n",
        "      ) for i in range(num_examples)\n",
        "  ]\n",
        "\n",
        "\n",
        "#\n",
        "# Output schema for lf.query.\n",
        "#\n",
        "class Answer(pg.Object):\n",
        "  step_by_step_thoughts: list[str]\n",
        "  final_answer: int\n",
        "\n",
        "\n",
        "#\n",
        "# Example of an evaluation class.\n",
        "#\n",
        "class Gsm8k(lf.eval.v2.Evaluation):\n",
        "  \"\"\"Gsm8K evaluation.\"\"\"\n",
        "\n",
        "  # `inputs` is a field declared from `lf.eval.v2.Evaluation`.\n",
        "  inputs = gsm8k_inputs(num_examples=2)\n",
        "\n",
        "  # User argument.\n",
        "  lm: lf.LanguageModel\n",
        "\n",
        "  #\n",
        "  # Definition of the metric class.\n",
        "  #\n",
        "  class Match(lf.eval.v2.metrics.Match):\n",
        "\n",
        "    def match(self, example_input: pg.Dict, output: Answer) -\u003e bool:\n",
        "      return example_input.answer == output.final_answer\n",
        "\n",
        "  # Specification for which metrics to use.\n",
        "  metrics = [Match()]\n",
        "\n",
        "  # The processing logic, which maps an item from input functor to an output\n",
        "  # which will be used for metric computation.\n",
        "  def process(\n",
        "      self,\n",
        "      example: lf.eval.v2.Example\n",
        "  ) -\u003e tuple[\n",
        "      Answer,              # Output for metric to process.\n",
        "      dict[str, Any]       # Metadata for display.\n",
        "  ]:\n",
        "    with lf.track_queries() as queries:\n",
        "      answer = lf.query(\n",
        "          example.input.question,\n",
        "          Answer,\n",
        "          lm=self.lm\n",
        "      )\n",
        "    return answer, dict(queries=queries)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 256
        },
        "id": "7BHQK88DJ7gk",
        "outputId": "8e217a5b-f74b-4960-eafb-bb708e1b0b95"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\u003chtml\u003e\n",
              "\u003chead\u003e\n",
              "\u003cstyle\u003e\n",
              "span.tooltip {\n",
              "  visibility: hidden;\n",
              "  white-space: pre-wrap;\n",
              "  font-weight: normal;\n",
              "  background-color: #484848;\n",
              "  color: #fff;\n",
              "  padding: 10px;\n",
              "  border-radius: 6px;\n",
              "  position: absolute;\n",
              "  z-index: 1;\n",
              "}\n",
              "span.tooltip:hover {\n",
              "  visibility: visible;\n",
              "}\n",
              ".tooltip.html-content {\n",
              "  white-space: inherit;\n",
              "  background-color: white;\n",
              "  color: inherit;\n",
              "  box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\n",
              "}\n",
              ".label:hover + .tooltip {\n",
              "  visibility: visible;\n",
              "}\n",
              ".label {\n",
              "  display: inline-block;\n",
              "  color: inherit;\n",
              "  padding: 5px;\n",
              "}\n",
              ".label-container {\n",
              "    display: inline-block;\n",
              "}\n",
              ".progress-bar {\n",
              "    display: inline-block;\n",
              "}\n",
              ".progress-bar .shade {\n",
              "  position: relative;\n",
              "  background-color: #EEE;\n",
              "  display: inline-flex;\n",
              "  margin: 10px;\n",
              "  width: 100px; \n",
              "  height: 5px;\n",
              "  border-radius: 5px; \n",
              "  overflow: hidden;\n",
              "}\n",
              ".progress-bar .sub-progress {\n",
              "  background-color: dodgerblue;\n",
              "  height: 10px;\n",
              "  width: 0%;\n",
              "  border-radius: 0px;\n",
              "}\n",
              ".eval-progress {\n",
              "  display: inline-block;\n",
              "}\n",
              ".sub-progress.skipped {\n",
              "  background-color:yellow;\n",
              "}\n",
              ".sub-progress.processed {\n",
              "  background-color:#00B000;\n",
              "}\n",
              ".sub-progress.failed {\n",
              "  background-color:red;\n",
              "}\n",
              ".progress-time {\n",
              "  font-weight: normal;\n",
              "  margin-left: 10px;\n",
              "  border-radius: 5px;\n",
              "  color: #CCC;\n",
              "  padding: 5px;\n",
              "}\n",
              ".badge {\n",
              "  background-color: #EEE;\n",
              "  border-radius: 5px;\n",
              "  color: #777;\n",
              "}\n",
              ".usage-summary.label {\n",
              "    display: inline-flex;\n",
              "    border-radius: 5px;\n",
              "    padding: 5px;\n",
              "    background-color: #f1f1f1;\n",
              "    color: #CCC;\n",
              "}\n",
              ".usage-summary.label::before {\n",
              "    content: '$';\n",
              "}\n",
              "/* Summary styles. */\n",
              "details.pyglove summary {\n",
              "  font-weight: bold;\n",
              "  margin: -0.5em -0.5em 0;\n",
              "  padding: 0.5em;\n",
              "}\n",
              ".summary-name {\n",
              "  display: inline;\n",
              "  padding: 3px 5px 3px 5px;\n",
              "  margin: 0 5px;\n",
              "  border-radius: 3px;\n",
              "}\n",
              ".summary-title {\n",
              "  display: inline;\n",
              "}\n",
              ".summary-name + div.summary-title {\n",
              "  display: inline;\n",
              "  color: #aaa;\n",
              "}\n",
              ".summary-title:hover + span.tooltip {\n",
              "  visibility: visible;\n",
              "}\n",
              ".summary-name:hover \u003e span.tooltip {\n",
              "  visibility: visible;\n",
              "  background-color: darkblue;\n",
              "}\n",
              ".label-group {\n",
              "  display: inline-flex;\n",
              "  overflow: hidden;\n",
              "  border-radius: 5px;\n",
              "  border: 1px solid #DDD;\n",
              "  padding: 0px;\n",
              "  margin: 5px\n",
              "}\n",
              ".label-group .group-name {\n",
              "  padding: 5px;\n",
              "  color: white;\n",
              "  background-color: dodgerblue;\n",
              "}\n",
              ".label-group .group-name \u003e .text {\n",
              "  color: white;\n",
              "}\n",
              ".label-group .group-value {\n",
              "}\n",
              ".label-group .badge {\n",
              "  border-radius: 0px;\n",
              "}\n",
              ".metric-container {\n",
              "  display: inline-flex;\n",
              "  overflow: hidden;\n",
              "  border-radius: 5px;\n",
              "  border: 0px;\n",
              "  margin: 5px;\n",
              "  padding: 0px;\n",
              "}\n",
              ".metric-container .label-container {\n",
              "  vertical-align: middle;\n",
              "}\n",
              ".metric-value.oop_errors {\n",
              "  color: magenta;\n",
              "  background-color: #f9e6eb;\n",
              "}\n",
              ".metric-value.non_oop_errors {\n",
              "  color: red;\n",
              "  background-color: #fdcccc;\n",
              "}\n",
              ".metric-name.match {\n",
              "  padding: 5px;\n",
              "  color: white;\n",
              "  background-color: purple;\n",
              "}\n",
              ".metric-value.matches {\n",
              "  color: green;\n",
              "  background-color: #dcefbe;\n",
              "}\n",
              ".metric-value.mismatches {\n",
              "  color: orange;\n",
              "  background-color: #ffefc4;\n",
              "}\n",
              "/* Value details styles. */\n",
              "details.pyglove {\n",
              "  border: 1px solid #aaa;\n",
              "  border-radius: 4px;\n",
              "  padding: 0.5em 0.5em 0;\n",
              "  margin: 0.25em 0;\n",
              "}\n",
              "details.pyglove[open] {\n",
              "  padding: 0.5em 0.5em 0.5em;\n",
              "}\n",
              ".highlight {\n",
              "  background-color: Mark;\n",
              "}\n",
              ".lowlight {\n",
              "  opacity: 0.2;\n",
              "}\n",
              ".experiment-summary {\n",
              "    display: inline-block;\n",
              "    font-weight: normal;\n",
              "}\n",
              ".experiment-name {\n",
              "    font-weight: bold;\n",
              "}\n",
              ".experiment-dir.label {\n",
              "    color: revert;\n",
              "    margin-left: 0px;\n",
              "    padding: 2px;\n",
              "}\n",
              ".usage-summary-badge {\n",
              "  margin-left: 10px;\n",
              "}\n",
              "body {\n",
              "  font: normal 16px \"Roboto\",\"Noto\",sans-serif;\n",
              "}\n",
              "details.eval-card {\n",
              "  display: inline-block;\n",
              "  border: 0px;\n",
              "  box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\n",
              "  margin: 15px;\n",
              "}\n",
              ".eval-card details {\n",
              "  border: 0px;\n",
              "}\n",
              ".badge-groups {\n",
              "  font-weight: normal;\n",
              "  padding: 5px;\n",
              "}\n",
              ".parameter-group {\n",
              "  display: inline-grid;\n",
              "  grid-template-rows: auto auto;\n",
              "  border: 0px;\n",
              "  margin-right: 10px;\n",
              "}\n",
              ".parameter.badge {\n",
              "  margin: 2px;\n",
              "}\n",
              ".metric-group {\n",
              "  display: inline-grid;\n",
              "  grid-template-rows: auto auto;\n",
              "}\n",
              ".eval-details .progress-bar \u003e .shade {\n",
              "  visibility: hidden;\n",
              "  width: 0px;\n",
              "  margin: 0px;\n",
              "}\n",
              ".eval-details .progress-label {\n",
              "  font-size: 16px;\n",
              "  background-color: #eee;\n",
              "}\n",
              ".eval-details .progress-time {\n",
              "  font-size: 16px;\n",
              "  color: dodgerblue;\n",
              "  background-color: #eee;\n",
              "  margin-right: 10px;\n",
              "}\n",
              ".eval-details .usage-summary.badge {\n",
              "  color: orange;\n",
              "  font-size: 16px;\n",
              "  background-color: #eee;\n",
              "}\n",
              ".eval-details .experiment-links {\n",
              "  display: block;\n",
              "  border: 0px;\n",
              "  margin: 0px;\n",
              "}\n",
              ".eval-details .tab-control {\n",
              "  width: 100%;\n",
              "}\n",
              ".eval-details .tab-button {\n",
              "  font-size: large;\n",
              "}\n",
              ".experiment-links .label {\n",
              "  color: revert;\n",
              "  margin: 0px;\n",
              "  padding: 2px;\n",
              "}\n",
              ".eval-details .experiment-id {\n",
              "  font-size: 2.0em;\n",
              "  font-weight: bold;\n",
              "  display: block;\n",
              "}\n",
              ".eval-details .parameter-group {\n",
              "  display: inline-block;\n",
              "  padding: 5px;\n",
              "}\n",
              ".eval-definition {\n",
              "  white-space: pre;\n",
              "  background-color: #eee;\n",
              "  padding: 15px;\n",
              "}\n",
              ".eval-details .metric-container {\n",
              "  display: block;\n",
              "  padding: 15px 0px;\n",
              "}\n",
              ".example-link {\n",
              "  color: revert;\n",
              "}\n",
              ".example-view {\n",
              "  border: 0px;\n",
              "  width:100%;\n",
              "  height:100%;\n",
              "}\n",
              ".logs-textarea {\n",
              "  width: 100%;\n",
              "  height: 500px;\n",
              "  padding: 5px;\n",
              "  border: 1px solid #DDD;\n",
              "  background-color: #EEE;\n",
              "  resize: vertical;\n",
              "}\n",
              "\u003c/style\u003e\n",
              "\u003c/head\u003e\n",
              "\u003cbody\u003e\n",
              "\u003cdetails open class=\"pyglove gsm8k\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003e\u003cdiv class=\"experiment-summary\"\u003e\u003cdiv class=\"label-container\"\u003e\u003ca class=\"label experiment-name\" href=\"file:///tmp/test_run/run_20250122_6/summary.html\"\u003e20250122_6\u003c/a\u003e\u003cspan class=\"tooltip\"\u003eGsm8k(\n",
              "  lm = OneOf(\n",
              "    candidates = [\n",
              "      0 : Gpt4oMini(\n",
              "        cache = None\n",
              "      ),\n",
              "      1 : GptO1Mini(\n",
              "        cache = None\n",
              "      )\n",
              "    ]\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003ca class=\"label experiment-dir\" href=\"file:///tmp/test_run/run_20250122_6/\"\u003e[dir]\u003c/a\u003e\u003cdiv class=\"eval-progress\"\u003e\u003cdiv class=\"progress-bar\"\u003e\u003cdiv class=\"shade\"\u003e\u003cdiv class=\"sub-progress skipped\" id=\"control-138085542479376\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress processed\" id=\"control-138085542487696\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress failed\" id=\"control-138085543074256\"\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-label\" id=\"control-138085542926352\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542487248\"\u003eNot started\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-time not-started\" id=\"control-138085543852880\"\u003e00:00:00\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542484688\"\u003e{\n",
              "  duration = \u0026#x27;00:00:00\u0026#x27;,\n",
              "  last_update = \u0026#x27;2025/01/22 18:18:23\u0026#x27;,\n",
              "  start_time = None,\n",
              "  stop_time = None\n",
              "}\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138086135646672\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085543852560\"\u003eUsageSummary(\n",
              "  cached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  ),\n",
              "  uncached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/summary\u003e\u003cdiv\u003e\u003cdetails open class=\"pyglove gsm8k eval-card\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name eval-card\"\u003e#1\u003c/div\u003e\u003cdiv class=\"summary-title eval-card\"\u003e\u003cdiv class=\"experiment-summary\"\u003e\u003cdiv class=\"label-container\"\u003e\u003ca class=\"label experiment-name\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/index.html\"\u003eGsm8k@40316ddb\u003c/a\u003e\u003cspan class=\"tooltip\"\u003eGsm8k(\n",
              "  lm = Gpt4oMini(\n",
              "    cache = None\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003ca class=\"label experiment-dir\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/\"\u003e[dir]\u003c/a\u003e\u003cdiv class=\"eval-progress\"\u003e\u003cdiv class=\"progress-bar\"\u003e\u003cdiv class=\"shade\"\u003e\u003cdiv class=\"sub-progress skipped\" id=\"control-138085544068816\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress processed\" id=\"control-138086575878288\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress failed\" id=\"control-138086575869328\"\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-label\" id=\"control-138086575868560\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085543990544\"\u003eNot started\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-time not-started\" id=\"control-138085969422672\"\u003e00:00:00\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085553463632\"\u003e{\n",
              "  duration = \u0026#x27;00:00:00\u0026#x27;,\n",
              "  last_update = \u0026#x27;2025/01/22 18:18:23\u0026#x27;,\n",
              "  start_time = None,\n",
              "  stop_time = None\n",
              "}\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138086575879440\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085747146320\"\u003eUsageSummary(\n",
              "  cached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  ),\n",
              "  uncached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/summary\u003e\u003cdiv class=\"badge-groups\"\u003e\u003cdiv class=\"label-group parameter-group\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label parameter badge group-value\"\u003elm: Gpt4oMini(cache=None)\u003c/span\u003e\u003cspan class=\"tooltip\"\u003elm: Gpt4oMini(cache=None)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"metric-group\"\u003e\u003cdiv class=\"label-group metric-container\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label metric-name match group-name\"\u003eM\u003c/span\u003e\u003cspan class=\"tooltip\"\u003e__main__.Gsm8k.Match\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label matches metric-value group-value\" id=\"control-138085544080912\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085543599952\"\u003ematches: n/a\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label mismatches metric-value group-value\" id=\"control-138085544069072\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138086575869840\"\u003emismatches: n/a\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label oop_errors metric-value group-value\" id=\"control-138085544073104\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085546139472\"\u003eoop_errors: n/a\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label non_oop_errors metric-value group-value\" id=\"control-138085542759376\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138086575809808\"\u003enon_oop_errors: n/a\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/details\u003e\u003cdetails open class=\"pyglove gsm8k eval-card\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name eval-card\"\u003e#2\u003c/div\u003e\u003cdiv class=\"summary-title eval-card\"\u003e\u003cdiv class=\"experiment-summary\"\u003e\u003cdiv class=\"label-container\"\u003e\u003ca class=\"label experiment-name\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/c0a40a85/index.html\"\u003eGsm8k@c0a40a85\u003c/a\u003e\u003cspan class=\"tooltip\"\u003eGsm8k(\n",
              "  lm = GptO1Mini(\n",
              "    cache = None\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003ca class=\"label experiment-dir\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/c0a40a85/\"\u003e[dir]\u003c/a\u003e\u003cdiv class=\"eval-progress\"\u003e\u003cdiv class=\"progress-bar\"\u003e\u003cdiv class=\"shade\"\u003e\u003cdiv class=\"sub-progress skipped\" id=\"control-138085542984848\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress processed\" id=\"control-138085553462672\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress failed\" id=\"control-138085546177296\"\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-label\" id=\"control-138085542458384\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542744464\"\u003eNot started\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-time not-started\" id=\"control-138085543076304\"\u003e00:00:00\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542743760\"\u003e{\n",
              "  duration = \u0026#x27;00:00:00\u0026#x27;,\n",
              "  last_update = \u0026#x27;2025/01/22 18:18:24\u0026#x27;,\n",
              "  start_time = None,\n",
              "  stop_time = None\n",
              "}\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138085999228368\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085970066768\"\u003eUsageSummary(\n",
              "  cached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  ),\n",
              "  uncached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/summary\u003e\u003cdiv class=\"badge-groups\"\u003e\u003cdiv class=\"label-group parameter-group\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label parameter badge group-value\"\u003elm: GptO1Mini(cache=None)\u003c/span\u003e\u003cspan class=\"tooltip\"\u003elm: GptO1Mini(cache=None)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"metric-group\"\u003e\u003cdiv class=\"label-group metric-container\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label metric-name match group-name\"\u003eM\u003c/span\u003e\u003cspan class=\"tooltip\"\u003e__main__.Gsm8k.Match\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label matches metric-value group-value\" id=\"control-138085697596944\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542749008\"\u003ematches: n/a\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label mismatches metric-value group-value\" id=\"control-138085999227088\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542483664\"\u003emismatches: n/a\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label oop_errors metric-value group-value\" id=\"control-138085542612240\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542982800\"\u003eoop_errors: n/a\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label non_oop_errors metric-value group-value\" id=\"control-138085542615888\"\u003en/a\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085542969680\"\u003enon_oop_errors: n/a\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\n",
              "\u003c/body\u003e\n",
              "\u003c/html\u003e"
            ],
            "text/plain": [
              "Gsm8k(progress=Progress(num_total=None, num_processed=0, num_failed=0, num_skipped=0, start_time=None, stop_time=None, execution_summary=TimeIt.StatusSummary(breakdown={})), usage_summary=UsageSummary(cached=AggregatedUsage(total=LMSamplingUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0, num_requests=0, estimated_cost=0.0, retry_stats=RetryStats(num_occurences=0, total_wait_interval=0.0, total_call_interval=0.0, errors={})), breakdown={}), uncached=AggregatedUsage(total=LMSamplingUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0, num_requests=0, estimated_cost=0.0, retry_stats=RetryStats(num_occurences=0, total_wait_interval=0.0, total_call_interval=0.0, errors={})), breakdown={})), plugins=[], inputs=gsm8k_inputs(num_examples=2), metrics=[0: Match(name='match', oop_errors=Rate(data_points=[], total=0), non_oop_errors=Rate(data_points=[], total=0), matches=Rate(data_points=[], total=0), mismatches=Rate(data_points=[], total=0))], max_workers=32, lm=OneOf(name=None, hints=None, num_choices=1, candidates=[0: Gpt4oMini(sampling_options=LMSamplingOptions(temperature=None, max_tokens=None, n=1, top_k=40, top_p=None, stop=None, random_seed=None, logprobs=False, top_logprobs=None), cache=None, max_concurrency=None, timeout=120.0, max_attempts=5, retry_interval=(5, 60), exponential_backoff=True, max_retry_interval=300, debug=False, api_endpoint='https://api.openai.com/v1/chat/completions', model='gpt-4o-mini', headers=None, multimodal=True, api_key=None, organization=None, project=None), 1: GptO1Mini(sampling_options=LMSamplingOptions(temperature=None, max_tokens=None, n=1, top_k=40, top_p=None, stop=None, random_seed=None, logprobs=False, top_logprobs=None), cache=None, max_concurrency=None, timeout=None, max_attempts=5, retry_interval=(5, 60), exponential_backoff=True, max_retry_interval=300, debug=False, api_endpoint='https://api.openai.com/v1/chat/completions', model='o1-mini', headers=None, multimodal=False, api_key=None, organization=None, project=None)], choices_distinct=True, choices_sorted=False, where=None))"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542926352\");\n",
              "elem.textContent = \" 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487248\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 0.0% (0/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487248\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.textContent = \"0:00:00\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:00',\\n  last_update = '2025/01/22 18:18:24',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542458384\");\n",
              "elem.textContent = \" 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575868560\");\n",
              "elem.textContent = \" 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542744464\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 0.0% (0/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543990544\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 0.0% (0/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542744464\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543990544\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.textContent = \"0:00:00\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.textContent = \"0:00:00\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:00',\\n  last_update = '2025/01/22 18:18:24',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:00',\\n  last_update = '2025/01/22 18:18:24',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Creating a new cache as cache file '/tmp/test_run/run_20250122_6/Gsm8k/40316ddb/cache.json' does not exist.\n",
            "WARNING:root:Creating a new cache as cache file '/tmp/test_run/run_20250122_6/Gsm8k/c0a40a85/cache.json' does not exist.\n"
          ]
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085544080912\");\n",
              "elem.textContent = \"100.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543599952\");\n",
              "elem.textContent = \"matches: 100.0% (1/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543599952\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085544069072\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575869840\");\n",
              "elem.textContent = \"mismatches: 0.0% (0/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575869840\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085544073104\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085546139472\");\n",
              "elem.textContent = \"oop_errors: 0.0% (0/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085546139472\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542759376\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575809808\");\n",
              "elem.textContent = \"non_oop_errors: 0.0% (0/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575809808\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575878288\");\n",
              "elem.style = \"width:50%;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575868560\");\n",
              "elem.textContent = \" 50.0% (1/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543990544\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 50.0% (1/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543990544\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.textContent = \"0:00:04\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:04',\\n  last_update = '2025/01/22 18:18:28',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575879440\");\n",
              "elem.textContent = \"0.000\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575879440\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085747146320\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 144,\\n      completion_tokens = 85,\\n      total_tokens = 229,\\n      num_requests = 1,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 2.720,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 144,\\n        completion_tokens = 85,\\n        total_tokens = 229,\\n        num_requests = 1,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 2.720,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085747146320\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.textContent = \"0.000\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 144,\\n      completion_tokens = 85,\\n      total_tokens = 229,\\n      num_requests = 1,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 2.720,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 144,\\n        completion_tokens = 85,\\n        total_tokens = 229,\\n        num_requests = 1,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 2.720,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085544080912\");\n",
              "elem.textContent = \"100.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543599952\");\n",
              "elem.textContent = \"matches: 100.0% (2/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543599952\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085544069072\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575869840\");\n",
              "elem.textContent = \"mismatches: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575869840\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085544073104\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085546139472\");\n",
              "elem.textContent = \"oop_errors: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085546139472\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542759376\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575809808\");\n",
              "elem.textContent = \"non_oop_errors: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575809808\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575878288\");\n",
              "elem.style = \"width:100%;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575868560\");\n",
              "elem.textContent = \" 100.0% (2/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543990544\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 100.0% (2/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543990544\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.textContent = \"0:00:04\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:04',\\n  last_update = '2025/01/22 18:18:28',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None,\\n  execution = {\\n    evaluate = {\\n      num_started = 1,\\n      num_ended = 1,\\n      num_failed = 0,\\n      avg_duration = 2.85\\n    },\\n    evaluate.process = {\\n      num_started = 1,\\n      num_ended = 1,\\n      num_failed = 0,\\n      avg_duration = 2.81\\n    },\\n    evaluate.metric = {\\n      num_started = 1,\\n      num_ended = 1,\\n      num_failed = 0,\\n      avg_duration = 0.04\\n    }\\n  }\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575879440\");\n",
              "elem.textContent = \"0.000\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086575879440\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085747146320\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 324,\\n      completion_tokens = 221,\\n      total_tokens = 545,\\n      num_requests = 2,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 5.603,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 324,\\n        completion_tokens = 221,\\n        total_tokens = 545,\\n        num_requests = 2,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 5.603,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085747146320\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.textContent = \"0.000\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 324,\\n      completion_tokens = 221,\\n      total_tokens = 545,\\n      num_requests = 2,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 5.603,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 324,\\n        completion_tokens = 221,\\n        total_tokens = 545,\\n        num_requests = 2,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 5.603,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.textContent = \"0:00:04\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085969422672\");\n",
              "elem.style = \"color:#ccc;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:04',\\n  last_update = '2025/01/22 18:18:29',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = '2025/01/22 18:18:29',\\n  execution = {\\n    evaluate = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 3.0\\n    },\\n    evaluate.process = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 2.92\\n    },\\n    evaluate.metric = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 0.08\\n    }\\n  }\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553463632\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487696\");\n",
              "elem.style = \"width:50%;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542926352\");\n",
              "elem.textContent = \" 50.0% (1/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487248\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 50.0% (1/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487248\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.textContent = \"0:00:04\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:04',\\n  last_update = '2025/01/22 18:18:29',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085697596944\");\n",
              "elem.textContent = \"100.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542749008\");\n",
              "elem.textContent = \"matches: 100.0% (1/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542749008\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085999227088\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542483664\");\n",
              "elem.textContent = \"mismatches: 0.0% (0/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542483664\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542612240\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542982800\");\n",
              "elem.textContent = \"oop_errors: 0.0% (0/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542982800\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542615888\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542969680\");\n",
              "elem.textContent = \"non_oop_errors: 0.0% (0/1)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542969680\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553462672\");\n",
              "elem.style = \"width:50%;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542458384\");\n",
              "elem.textContent = \" 50.0% (1/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542744464\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 50.0% (1/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542744464\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.textContent = \"0:00:10\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:10',\\n  last_update = '2025/01/22 18:18:35',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085999228368\");\n",
              "elem.textContent = \"0.008\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085999228368\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085970066768\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 152,\\n      completion_tokens = 667,\\n      total_tokens = 819,\\n      num_requests = 1,\\n      estimated_cost = 0.008,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 6.989,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(o1-mini) = LMSamplingUsage(\\n        prompt_tokens = 152,\\n        completion_tokens = 667,\\n        total_tokens = 819,\\n        num_requests = 1,\\n        estimated_cost = 0.008,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 6.989,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085970066768\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.textContent = \"0.009\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 476,\\n      completion_tokens = 888,\\n      total_tokens = 1,364,\\n      num_requests = 3,\\n      estimated_cost = 0.009,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 12.592,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 324,\\n        completion_tokens = 221,\\n        total_tokens = 545,\\n        num_requests = 2,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 5.603,\\n          errors = {}\\n        )\\n      ),\\n      OpenAI(o1-mini) = LMSamplingUsage(\\n        prompt_tokens = 152,\\n        completion_tokens = 667,\\n        total_tokens = 819,\\n        num_requests = 1,\\n        estimated_cost = 0.008,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 6.989,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085697596944\");\n",
              "elem.textContent = \"100.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542749008\");\n",
              "elem.textContent = \"matches: 100.0% (2/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542749008\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085999227088\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542483664\");\n",
              "elem.textContent = \"mismatches: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542483664\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542612240\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542982800\");\n",
              "elem.textContent = \"oop_errors: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542982800\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542615888\");\n",
              "elem.textContent = \"0.0%\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542969680\");\n",
              "elem.textContent = \"non_oop_errors: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542969680\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085553462672\");\n",
              "elem.style = \"width:100%;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542458384\");\n",
              "elem.textContent = \" 100.0% (2/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542744464\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 100.0% (2/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542744464\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.textContent = \"0:00:12\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:12',\\n  last_update = '2025/01/22 18:18:36',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None,\\n  execution = {\\n    evaluate = {\\n      num_started = 1,\\n      num_ended = 1,\\n      num_failed = 0,\\n      avg_duration = 7.14\\n    },\\n    evaluate.process = {\\n      num_started = 1,\\n      num_ended = 1,\\n      num_failed = 0,\\n      avg_duration = 7.09\\n    },\\n    evaluate.metric = {\\n      num_started = 1,\\n      num_ended = 1,\\n      num_failed = 0,\\n      avg_duration = 0.04\\n    }\\n  }\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085999228368\");\n",
              "elem.textContent = \"0.026\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085999228368\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085970066768\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 340,\\n      completion_tokens = 2,090,\\n      total_tokens = 2,430,\\n      num_requests = 2,\\n      estimated_cost = 0.026,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 15.508,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(o1-mini) = LMSamplingUsage(\\n        prompt_tokens = 340,\\n        completion_tokens = 2,090,\\n        total_tokens = 2,430,\\n        num_requests = 2,\\n        estimated_cost = 0.026,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 15.508,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085970066768\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.textContent = \"0.026\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138086135646672\");\n",
              "elem.style = \"None\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.textContent = \"UsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 664,\\n      completion_tokens = 2,311,\\n      total_tokens = 2,975,\\n      num_requests = 4,\\n      estimated_cost = 0.026,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 21.110,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 324,\\n        completion_tokens = 221,\\n        total_tokens = 545,\\n        num_requests = 2,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 5.603,\\n          errors = {}\\n        )\\n      ),\\n      OpenAI(o1-mini) = LMSamplingUsage(\\n        prompt_tokens = 340,\\n        completion_tokens = 2,090,\\n        total_tokens = 2,430,\\n        num_requests = 2,\\n        estimated_cost = 0.026,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 15.508,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852560\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.textContent = \"0:00:12\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543076304\");\n",
              "elem.style = \"color:#ccc;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:12',\\n  last_update = '2025/01/22 18:18:36',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = '2025/01/22 18:18:36',\\n  execution = {\\n    evaluate = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 7.89\\n    },\\n    evaluate.process = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 7.85\\n    },\\n    evaluate.metric = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 0.04\\n    }\\n  }\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542743760\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487696\");\n",
              "elem.style = \"width:100%;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542926352\");\n",
              "elem.textContent = \" 100.0% (2/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487248\");\n",
              "elem.textContent = \"Skipped: 0.0% (0/2)\\nProcessed: 100.0% (2/2)\\nFailed: 0.0% (0/2)\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542487248\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.textContent = \"0:00:12\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.style = \"color:dodgerblue;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:12',\\n  last_update = '2025/01/22 18:18:36',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = None\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.textContent = \"0:00:12\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085543852880\");\n",
              "elem.style = \"color:#ccc;\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.textContent = \"{\\n  duration = '0:00:12',\\n  last_update = '2025/01/22 18:18:37',\\n  start_time = '2025/01/22 18:18:24',\\n  stop_time = '2025/01/22 18:18:37'\\n}\";"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "elem = document.getElementById(\"control-138085542484688\");\n",
              "elem.classList.remove(\"html-content\");"
            ],
            "text/plain": [
              "\u003cIPython.core.display.Javascript object\u003e"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import os\n",
        "\n",
        "os.environ['OPENAI_API_KEY'] = '\u003cUpdate this field with your OpenAI key\u003e'\n",
        "\n",
        "benchmark = Gsm8k(\n",
        "    # Use `pg.oneof` to enable benchmark sweeping, which runs multiple evaluations\n",
        "    # in parallel.\n",
        "    lm=pg.oneof([\n",
        "        lf.llms.Gpt4oMini(),\n",
        "        lf.llms.GptO1Mini()\n",
        "    ])\n",
        ")\n",
        "root_dir = '/tmp/test_run'\n",
        "run = benchmark.run(root_dir, 'new')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "he_qyIAfPUiU"
      },
      "source": [
        "# Study the evaluation details of a dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        },
        "id": "LAipO2yUOgFX",
        "outputId": "95a925ac-c98a-48b3-ad50-542024c308ed"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\u003chtml\u003e\n",
              "\u003chead\u003e\n",
              "\u003cstyle\u003e\n",
              ".label {\n",
              "  display: inline-block;\n",
              "  color: inherit;\n",
              "  padding: 5px;\n",
              "}\n",
              ".label-container {\n",
              "    display: inline-block;\n",
              "}\n",
              ".label-group {\n",
              "  display: inline-flex;\n",
              "  overflow: hidden;\n",
              "  border-radius: 5px;\n",
              "  border: 1px solid #DDD;\n",
              "  padding: 0px;\n",
              "  margin: 5px\n",
              "}\n",
              ".label-group .group-name {\n",
              "  padding: 5px;\n",
              "  color: white;\n",
              "  background-color: dodgerblue;\n",
              "}\n",
              ".label-group .group-name \u003e .text {\n",
              "  color: white;\n",
              "}\n",
              ".label-group .group-value {\n",
              "}\n",
              ".label-group .badge {\n",
              "  border-radius: 0px;\n",
              "}\n",
              "span.tooltip {\n",
              "  visibility: hidden;\n",
              "  white-space: pre-wrap;\n",
              "  font-weight: normal;\n",
              "  background-color: #484848;\n",
              "  color: #fff;\n",
              "  padding: 10px;\n",
              "  border-radius: 6px;\n",
              "  position: absolute;\n",
              "  z-index: 1;\n",
              "}\n",
              "span.tooltip:hover {\n",
              "  visibility: visible;\n",
              "}\n",
              ".tooltip.html-content {\n",
              "  white-space: inherit;\n",
              "  background-color: white;\n",
              "  color: inherit;\n",
              "  box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\n",
              "}\n",
              ".label:hover + .tooltip {\n",
              "  visibility: visible;\n",
              "}\n",
              ".progress-bar {\n",
              "    display: inline-block;\n",
              "}\n",
              ".progress-bar .shade {\n",
              "  position: relative;\n",
              "  background-color: #EEE;\n",
              "  display: inline-flex;\n",
              "  margin: 10px;\n",
              "  width: 100px; \n",
              "  height: 5px;\n",
              "  border-radius: 5px; \n",
              "  overflow: hidden;\n",
              "}\n",
              ".progress-bar .sub-progress {\n",
              "  background-color: dodgerblue;\n",
              "  height: 10px;\n",
              "  width: 0%;\n",
              "  border-radius: 0px;\n",
              "}\n",
              ".eval-progress {\n",
              "  display: inline-block;\n",
              "}\n",
              ".sub-progress.skipped {\n",
              "  background-color:yellow;\n",
              "}\n",
              ".sub-progress.processed {\n",
              "  background-color:#00B000;\n",
              "}\n",
              ".sub-progress.failed {\n",
              "  background-color:red;\n",
              "}\n",
              ".progress-time {\n",
              "  font-weight: normal;\n",
              "  margin-left: 10px;\n",
              "  border-radius: 5px;\n",
              "  color: #CCC;\n",
              "  padding: 5px;\n",
              "}\n",
              ".badge {\n",
              "  background-color: #EEE;\n",
              "  border-radius: 5px;\n",
              "  color: #777;\n",
              "}\n",
              ".usage-summary.label {\n",
              "    display: inline-flex;\n",
              "    border-radius: 5px;\n",
              "    padding: 5px;\n",
              "    background-color: #f1f1f1;\n",
              "    color: #CCC;\n",
              "}\n",
              ".usage-summary.label::before {\n",
              "    content: '$';\n",
              "}\n",
              ".metric-container {\n",
              "  display: inline-flex;\n",
              "  overflow: hidden;\n",
              "  border-radius: 5px;\n",
              "  border: 0px;\n",
              "  margin: 5px;\n",
              "  padding: 0px;\n",
              "}\n",
              ".metric-container .label-container {\n",
              "  vertical-align: middle;\n",
              "}\n",
              ".metric-value.oop_errors {\n",
              "  color: magenta;\n",
              "  background-color: #f9e6eb;\n",
              "}\n",
              ".metric-value.non_oop_errors {\n",
              "  color: red;\n",
              "  background-color: #fdcccc;\n",
              "}\n",
              ".metric-name.match {\n",
              "  padding: 5px;\n",
              "  color: white;\n",
              "  background-color: purple;\n",
              "}\n",
              ".metric-value.matches {\n",
              "  color: green;\n",
              "  background-color: #dcefbe;\n",
              "}\n",
              ".metric-value.mismatches {\n",
              "  color: orange;\n",
              "  background-color: #ffefc4;\n",
              "}\n",
              ".tab-control {\n",
              "  border-spacing: 0px;\n",
              "  border-collapse: collapse;\n",
              "  margin-top: 10px;\n",
              "}\n",
              ".tab-control td {\n",
              "  padding: 0px;\n",
              "  margin: 0px;\n",
              "  vertical-align: top;\n",
              "}\n",
              ".top.tab-button-group {\n",
              "  border-left: 1px solid #DDD;\n",
              "  border-top: 1px solid #DDD;\n",
              "  border-right: 1px solid #DDD;\n",
              "  border-radius: 5px 5px 0px 0px;\n",
              "  padding: 0px 5px 0px 0px;\n",
              "  margin-bottom: -2px;\n",
              "}\n",
              ".tab-button {\n",
              "  background-color: transparent;\n",
              "  border-radius: 5px;\n",
              "  border: 0px;\n",
              "  font-weight: bold;\n",
              "  color: gray;\n",
              "  outline: none;\n",
              "  cursor: pointer;\n",
              "  transition: 0.3s;\n",
              "}\n",
              ".tab-button:hover {\n",
              "  background-color: #fff1dd;\n",
              "}\n",
              ".tab-button.selected {\n",
              "  background-color: #f0ecf9;\n",
              "  color: black;\n",
              "}\n",
              ".top.tab-content-group {\n",
              "  border-left: 1px solid #DDD;\n",
              "  border-right: 1px solid #DDD;\n",
              "  border-bottom: 1px solid #DDD;\n",
              "  border-radius: 0px 0px 5px 5px;\n",
              "  margin: 0px;\n",
              "  padding: 5px;\n",
              "  height: 100%;\n",
              "}\n",
              ".top \u003e .tab-button {\n",
              "  margin: 5px 0px 5px 5px;\n",
              "}\n",
              ".tab-content {\n",
              "  display: none;\n",
              "}\n",
              ".tab-content.selected {\n",
              "  display: block;\n",
              "}\n",
              ".left.tab-button-group {\n",
              "  display: inline-flex;\n",
              "  flex-direction: column;\n",
              "  border: 1px solid #DDD;\n",
              "  border-radius: 5px;\n",
              "  margin-right: 5px;\n",
              "  padding: 0px 0px 5px 0px;\n",
              "}\n",
              ".left.tab-content-group {\n",
              "  border: 0px\n",
              "  margin: 0px;\n",
              "  padding: 0px;\n",
              "  height: 100%;\n",
              "}\n",
              ".left \u003e .tab-button {\n",
              "  text-align: left;\n",
              "  margin: 5px 5px 0px 5px;\n",
              "}\n",
              ".experiment-summary {\n",
              "    display: inline-block;\n",
              "    font-weight: normal;\n",
              "}\n",
              ".experiment-name {\n",
              "    font-weight: bold;\n",
              "}\n",
              ".experiment-dir.label {\n",
              "    color: revert;\n",
              "    margin-left: 0px;\n",
              "    padding: 2px;\n",
              "}\n",
              ".usage-summary-badge {\n",
              "  margin-left: 10px;\n",
              "}\n",
              "body {\n",
              "  font: normal 16px \"Roboto\",\"Noto\",sans-serif;\n",
              "}\n",
              "details.eval-card {\n",
              "  display: inline-block;\n",
              "  border: 0px;\n",
              "  box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\n",
              "  margin: 15px;\n",
              "}\n",
              ".eval-card details {\n",
              "  border: 0px;\n",
              "}\n",
              ".badge-groups {\n",
              "  font-weight: normal;\n",
              "  padding: 5px;\n",
              "}\n",
              ".parameter-group {\n",
              "  display: inline-grid;\n",
              "  grid-template-rows: auto auto;\n",
              "  border: 0px;\n",
              "  margin-right: 10px;\n",
              "}\n",
              ".parameter.badge {\n",
              "  margin: 2px;\n",
              "}\n",
              ".metric-group {\n",
              "  display: inline-grid;\n",
              "  grid-template-rows: auto auto;\n",
              "}\n",
              ".eval-details .progress-bar \u003e .shade {\n",
              "  visibility: hidden;\n",
              "  width: 0px;\n",
              "  margin: 0px;\n",
              "}\n",
              ".eval-details .progress-label {\n",
              "  font-size: 16px;\n",
              "  background-color: #eee;\n",
              "}\n",
              ".eval-details .progress-time {\n",
              "  font-size: 16px;\n",
              "  color: dodgerblue;\n",
              "  background-color: #eee;\n",
              "  margin-right: 10px;\n",
              "}\n",
              ".eval-details .usage-summary.badge {\n",
              "  color: orange;\n",
              "  font-size: 16px;\n",
              "  background-color: #eee;\n",
              "}\n",
              ".eval-details .experiment-links {\n",
              "  display: block;\n",
              "  border: 0px;\n",
              "  margin: 0px;\n",
              "}\n",
              ".eval-details .tab-control {\n",
              "  width: 100%;\n",
              "}\n",
              ".eval-details .tab-button {\n",
              "  font-size: large;\n",
              "}\n",
              ".experiment-links .label {\n",
              "  color: revert;\n",
              "  margin: 0px;\n",
              "  padding: 2px;\n",
              "}\n",
              ".eval-details .experiment-id {\n",
              "  font-size: 2.0em;\n",
              "  font-weight: bold;\n",
              "  display: block;\n",
              "}\n",
              ".eval-details .parameter-group {\n",
              "  display: inline-block;\n",
              "  padding: 5px;\n",
              "}\n",
              ".eval-definition {\n",
              "  white-space: pre;\n",
              "  background-color: #eee;\n",
              "  padding: 15px;\n",
              "}\n",
              ".eval-details .metric-container {\n",
              "  display: block;\n",
              "  padding: 15px 0px;\n",
              "}\n",
              ".example-link {\n",
              "  color: revert;\n",
              "}\n",
              ".example-view {\n",
              "  border: 0px;\n",
              "  width:100%;\n",
              "  height:100%;\n",
              "}\n",
              ".logs-textarea {\n",
              "  width: 100%;\n",
              "  height: 500px;\n",
              "  padding: 5px;\n",
              "  border: 1px solid #DDD;\n",
              "  background-color: #EEE;\n",
              "  resize: vertical;\n",
              "}\n",
              "\u003c/style\u003e\n",
              "\u003cscript\u003e\n",
              "function openTab(event, controlId, tabId) {\n",
              "  const tabButtons = document.querySelectorAll('#' + controlId + '-button-group \u003e .tab-button');\n",
              "  for (let i = 0; i \u003c tabButtons.length; i++) {\n",
              "    tabButtons[i].classList.remove('selected');\n",
              "  }\n",
              "  const tabContents = document.querySelectorAll('#' + controlId + '-content-group \u003e .tab-content');\n",
              "  for (let i = 0; i \u003c tabContents.length; i++) {\n",
              "    tabContents[i].classList.remove('selected')\n",
              "  }\n",
              "  const tabButton = event.currentTarget;\n",
              "  tabButton.classList.add('selected');\n",
              "  document.getElementById(tabId).classList.add('selected');\n",
              "}\n",
              "\u003c/script\u003e\n",
              "\u003c/head\u003e\n",
              "\u003cbody\u003e\n",
              "\u003cdiv class=\"eval-details\"\u003e\u003cdiv\u003e\u003cdiv class=\"label-group experiment-links\"\u003e\u003ca class=\"label summary-link group-value\" href=\"file:///tmp/test_run/run_20250122_6/summary.html\"\u003eSummary\u003c/a\u003e\u003cspan class=\"label group-value\"\u003e|\u003c/span\u003e\u003ca class=\"label dir-link group-value\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/\"\u003eDirectory\u003c/a\u003e\u003c/div\u003e\u003cspan class=\"label experiment-id\"\u003eGsm8k@40316ddb\u003c/span\u003e\u003cdiv class=\"eval-progress\"\u003e\u003cdiv class=\"progress-bar\"\u003e\u003cdiv class=\"shade\"\u003e\u003cdiv class=\"sub-progress skipped\" style=\"width:0%;\" id=\"control-138085516307728\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress processed\" style=\"width:100%;\" id=\"control-138085560148560\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress failed\" style=\"width:0%;\" id=\"control-138085543052112\"\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-label\" id=\"control-138085516301776\"\u003e 100.0% (2/2)\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085516286480\"\u003eSkipped: 0.0% (0/2)\n",
              "Processed: 100.0% (2/2)\n",
              "Failed: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-time stopped\"\u003e0:00:04\u003c/span\u003e\u003cspan class=\"tooltip\"\u003e{\n",
              "  duration = \u0026#x27;0:00:04\u0026#x27;,\n",
              "  last_update = \u0026#x27;2025/01/22 18:18:29\u0026#x27;,\n",
              "  start_time = \u0026#x27;2025/01/22 18:18:24\u0026#x27;,\n",
              "  stop_time = \u0026#x27;2025/01/22 18:18:29\u0026#x27;,\n",
              "  execution = {\n",
              "    evaluate = {\n",
              "      num_started = 2,\n",
              "      num_ended = 2,\n",
              "      num_failed = 0,\n",
              "      avg_duration = 3.0\n",
              "    },\n",
              "    evaluate.process = {\n",
              "      num_started = 2,\n",
              "      num_ended = 2,\n",
              "      num_failed = 0,\n",
              "      avg_duration = 2.92\n",
              "    },\n",
              "    evaluate.metric = {\n",
              "      num_started = 2,\n",
              "      num_ended = 2,\n",
              "      num_failed = 0,\n",
              "      avg_duration = 0.08\n",
              "    }\n",
              "  }\n",
              "}\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138086575879440\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085747146320\"\u003eUsageSummary(\n",
              "  cached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  ),\n",
              "  uncached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 324,\n",
              "      completion_tokens = 221,\n",
              "      total_tokens = 545,\n",
              "      num_requests = 2,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 5.603,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {\n",
              "      OpenAI(gpt-4o-mini) = LMSamplingUsage(\n",
              "        prompt_tokens = 324,\n",
              "        completion_tokens = 221,\n",
              "        total_tokens = 545,\n",
              "        num_requests = 2,\n",
              "        estimated_cost = 0.000,\n",
              "        retry_stats = RetryStats(\n",
              "          num_occurences = 0,\n",
              "          total_wait_interval = 0.000,\n",
              "          total_call_interval = 5.603,\n",
              "          errors = {}\n",
              "        )\n",
              "      )\n",
              "    }\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-group parameter-group\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label parameter badge group-value\"\u003elm: Gpt4oMini(cache=None)\u003c/span\u003e\u003cspan class=\"tooltip\"\u003elm: Gpt4oMini(cache=None)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085665169296-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085665169296', 'control-138085665169296-0')\"\u003e\u003cspan class=\"label\"\u003eDefinition\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, 'control-138085665169296', 'control-138085665169296-1')\"\u003e\u003cspan class=\"label\"\u003eMetric: match\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085665169296', 'control-138085665169296-2')\"\u003e\u003cspan class=\"label\"\u003eLogs\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085665169296-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085665169296-0\"\u003e\u003cdiv\u003e\u003cspan class=\"label eval-definition\"\u003eGsm8k(\n",
              "  plugins = [],\n",
              "  inputs = gsm8k_inputs(\n",
              "    num_examples = 2\n",
              "  ),\n",
              "  metrics = [\n",
              "    0 : Match(\n",
              "      name = 'match',\n",
              "      oop_errors = 0.0%,\n",
              "      non_oop_errors = 0.0%,\n",
              "      matches = 100.0%,\n",
              "      mismatches = 0.0%\n",
              "    )\n",
              "  ],\n",
              "  max_workers = 32,\n",
              "  lm = Gpt4oMini(\n",
              "    sampling_options = LMSamplingOptions(\n",
              "      temperature = None,\n",
              "      max_tokens = None,\n",
              "      n = 1,\n",
              "      top_k = 40,\n",
              "      top_p = None,\n",
              "      stop = None,\n",
              "      random_seed = None,\n",
              "      logprobs = False,\n",
              "      top_logprobs = None\n",
              "    ),\n",
              "    cache = None,\n",
              "    max_concurrency = None,\n",
              "    timeout = 120.0,\n",
              "    max_attempts = 5,\n",
              "    retry_interval = (5, 60),\n",
              "    exponential_backoff = True,\n",
              "    max_retry_interval = 300,\n",
              "    debug = False,\n",
              "    api_endpoint = 'https://api.openai.com/v1/chat/completions',\n",
              "    model = 'gpt-4o-mini',\n",
              "    headers = None,\n",
              "    multimodal = True,\n",
              "    api_key = None,\n",
              "    organization = None,\n",
              "    project = None\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085665169296-1\"\u003e\u003cdiv\u003e\u003cdiv class=\"label-group metric-container\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label metric-name match group-name\"\u003eM\u003c/span\u003e\u003cspan class=\"tooltip\"\u003e__main__.Gsm8k.Match\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label matches metric-value group-value\" id=\"control-138085544080912\"\u003e100.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085543599952\"\u003ematches: 100.0% (2/2)\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label mismatches metric-value group-value\" id=\"control-138085544069072\"\u003e0.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138086575869840\"\u003emismatches: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label oop_errors metric-value group-value\" id=\"control-138085544073104\"\u003e0.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085546139472\"\u003eoop_errors: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label non_oop_errors metric-value group-value\" id=\"control-138085542759376\"\u003e0.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138086575809808\"\u003enon_oop_errors: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085545537488-button-group\"\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, 'control-138085545537488', 'control-138085545537488-0')\"\u003e\u003cspan class=\"label\"\u003ematches\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085545537488', 'control-138085545537488-1')\"\u003e\u003cspan class=\"label\"\u003emismatches\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085545537488', 'control-138085545537488-2')\"\u003e\u003cspan class=\"label\"\u003eoop_errors\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085545537488', 'control-138085545537488-3')\"\u003e\u003cspan class=\"label\"\u003enon_oop_errors\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085545537488-content-group\"\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085545537488-0\"\u003e\u003cdiv\u003e\u003ca class=\"label example-link\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/2.html\" target=\"example-view\"\u003e2\u003c/a\u003e\u003ca class=\"label example-link\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/1.html\" target=\"example-view\"\u003e1\u003c/a\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085545537488-1\"\u003e\u003cdiv\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085545537488-2\"\u003e\u003cdiv\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085545537488-3\"\u003e\u003cdiv\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085665169296-2\"\u003e\u003cdiv\u003e\u003ctextarea class=\"logs-textarea\" readonly=\"True\"\u003e18:18:23 INFO - Updated \u0026#x27;/tmp/test_run/run_20250122_6/Gsm8k/40316ddb/index.html\u0026#x27; in 0.26 seconds.\n",
              "18:18:24 INFO - Found 0 checkpoint files to load.\n",
              "18:18:24 INFO - No examples are loaded from checkpoint files. 2 examples will be processed from scratch. Example IDs: {1, 2}.\n",
              "18:18:24 INFO - Starting evaluation \u0026#x27;Gsm8k@40316ddb\u0026#x27; with 2 examples to evaluate.\n",
              "18:18:28 INFO - Example 2 checkpointed to /tmp/test_run/run_20250122_6/Gsm8k/40316ddb/checkpoint.bagz.\n",
              "18:18:29 INFO - Example 1 checkpointed to /tmp/test_run/run_20250122_6/Gsm8k/40316ddb/checkpoint.bagz.\n",
              "18:18:29 INFO - 2 examples are checkpointed to /tmp/test_run/run_20250122_6/Gsm8k/40316ddb/checkpoint.bagz.\n",
              "18:18:29 INFO - Gsm8k@40316ddb completed with 2 examples evaluated (0 from checkpoint, 2 newly processed).\n",
              "18:18:29 INFO - Updated \u0026#x27;/tmp/test_run/run_20250122_6/Gsm8k/40316ddb/index.html\u0026#x27; in 0.26 seconds.\n",
              "18:18:29 INFO - \u0026#x27;2.html\u0026#x27; generated in 1.27 seconds. \n",
              "18:18:30 INFO - \u0026#x27;1.html\u0026#x27; generated in 1.28 seconds. \u003c/textarea\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003ciframe class=\"example-view\" name=\"example-view\" src=\"./1.html\" title=\"Example view.\"\u003e\u003c/iframe\u003e\u003c/div\u003e\n",
              "\u003c/body\u003e\n",
              "\u003c/html\u003e"
            ],
            "text/plain": [
              "Html(content='\u003cdiv class=\"eval-details\"\u003e\u003cdiv\u003e\u003cdiv class=\"label-group experiment-links\"\u003e\u003ca class=\"label summary-link group-value\" href=\"file:///tmp/test_run/run_20250122_6/summary.html\"\u003eSummary\u003c/a\u003e\u003cspan class=\"label group-value\"\u003e|\u003c/span\u003e\u003ca class=\"label dir-link group-value\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/\"\u003eDirectory\u003c/a\u003e\u003c/div\u003e\u003cspan class=\"label experiment-id\"\u003eGsm8k@40316ddb\u003c/span\u003e\u003cdiv class=\"eval-progress\"\u003e\u003cdiv class=\"progress-bar\"\u003e\u003cdiv class=\"shade\"\u003e\u003cdiv class=\"sub-progress skipped\" style=\"width:0%;\" id=\"control-138085516307728\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress processed\" style=\"width:100%;\" id=\"control-138085560148560\"\u003e\u003c/div\u003e\u003cdiv class=\"sub-progress failed\" style=\"width:0%;\" id=\"control-138085543052112\"\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-label\" id=\"control-138085516301776\"\u003e 100.0% (2/2)\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085516286480\"\u003eSkipped: 0.0% (0/2)\\nProcessed: 100.0% (2/2)\\nFailed: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label progress-time stopped\"\u003e0:00:04\u003c/span\u003e\u003cspan class=\"tooltip\"\u003e{\\n  duration = \u0026#x27;0:00:04\u0026#x27;,\\n  last_update = \u0026#x27;2025/01/22 18:18:29\u0026#x27;,\\n  start_time = \u0026#x27;2025/01/22 18:18:24\u0026#x27;,\\n  stop_time = \u0026#x27;2025/01/22 18:18:29\u0026#x27;,\\n  execution = {\\n    evaluate = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 3.0\\n    },\\n    evaluate.process = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 2.92\\n    },\\n    evaluate.metric = {\\n      num_started = 2,\\n      num_ended = 2,\\n      num_failed = 0,\\n      avg_duration = 0.08\\n    }\\n  }\\n}\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138086575879440\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085747146320\"\u003eUsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 324,\\n      completion_tokens = 221,\\n      total_tokens = 545,\\n      num_requests = 2,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 5.603,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 324,\\n        completion_tokens = 221,\\n        total_tokens = 545,\\n        num_requests = 2,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 5.603,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"label-group parameter-group\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label parameter badge group-value\"\u003elm: Gpt4oMini(cache=None)\u003c/span\u003e\u003cspan class=\"tooltip\"\u003elm: Gpt4oMini(cache=None)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085665169296-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085665169296\\', \\'control-138085665169296-0\\')\"\u003e\u003cspan class=\"label\"\u003eDefinition\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, \\'control-138085665169296\\', \\'control-138085665169296-1\\')\"\u003e\u003cspan class=\"label\"\u003eMetric: match\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085665169296\\', \\'control-138085665169296-2\\')\"\u003e\u003cspan class=\"label\"\u003eLogs\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085665169296-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085665169296-0\"\u003e\u003cdiv\u003e\u003cspan class=\"label eval-definition\"\u003eGsm8k(\\n  plugins = [],\\n  inputs = gsm8k_inputs(\\n    num_examples = 2\\n  ),\\n  metrics = [\\n    0 : Match(\\n      name = \\'match\\',\\n      oop_errors = 0.0%,\\n      non_oop_errors = 0.0%,\\n      matches = 100.0%,\\n      mismatches = 0.0%\\n    )\\n  ],\\n  max_workers = 32,\\n  lm = Gpt4oMini(\\n    sampling_options = LMSamplingOptions(\\n      temperature = None,\\n      max_tokens = None,\\n      n = 1,\\n      top_k = 40,\\n      top_p = None,\\n      stop = None,\\n      random_seed = None,\\n      logprobs = False,\\n      top_logprobs = None\\n    ),\\n    cache = None,\\n    max_concurrency = None,\\n    timeout = 120.0,\\n    max_attempts = 5,\\n    retry_interval = (5, 60),\\n    exponential_backoff = True,\\n    max_retry_interval = 300,\\n    debug = False,\\n    api_endpoint = \\'https://api.openai.com/v1/chat/completions\\',\\n    model = \\'gpt-4o-mini\\',\\n    headers = None,\\n    multimodal = True,\\n    api_key = None,\\n    organization = None,\\n    project = None\\n  )\\n)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085665169296-1\"\u003e\u003cdiv\u003e\u003cdiv class=\"label-group metric-container\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label metric-name match group-name\"\u003eM\u003c/span\u003e\u003cspan class=\"tooltip\"\u003e__main__.Gsm8k.Match\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label matches metric-value group-value\" id=\"control-138085544080912\"\u003e100.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085543599952\"\u003ematches: 100.0% (2/2)\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label mismatches metric-value group-value\" id=\"control-138085544069072\"\u003e0.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138086575869840\"\u003emismatches: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label oop_errors metric-value group-value\" id=\"control-138085544073104\"\u003e0.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085546139472\"\u003eoop_errors: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label non_oop_errors metric-value group-value\" id=\"control-138085542759376\"\u003e0.0%\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138086575809808\"\u003enon_oop_errors: 0.0% (0/2)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085545537488-button-group\"\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, \\'control-138085545537488\\', \\'control-138085545537488-0\\')\"\u003e\u003cspan class=\"label\"\u003ematches\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085545537488\\', \\'control-138085545537488-1\\')\"\u003e\u003cspan class=\"label\"\u003emismatches\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085545537488\\', \\'control-138085545537488-2\\')\"\u003e\u003cspan class=\"label\"\u003eoop_errors\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085545537488\\', \\'control-138085545537488-3\\')\"\u003e\u003cspan class=\"label\"\u003enon_oop_errors\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085545537488-content-group\"\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085545537488-0\"\u003e\u003cdiv\u003e\u003ca class=\"label example-link\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/2.html\" target=\"example-view\"\u003e2\u003c/a\u003e\u003ca class=\"label example-link\" href=\"file:///tmp/test_run/run_20250122_6/Gsm8k/40316ddb/1.html\" target=\"example-view\"\u003e1\u003c/a\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085545537488-1\"\u003e\u003cdiv\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085545537488-2\"\u003e\u003cdiv\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085545537488-3\"\u003e\u003cdiv\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085665169296-2\"\u003e\u003cdiv\u003e\u003ctextarea class=\"logs-textarea\" readonly=\"True\"\u003e18:18:23 INFO - Updated \u0026#x27;/tmp/test_run/run_20250122_6/Gsm8k/40316ddb/index.html\u0026#x27; in 0.26 seconds.\\n18:18:24 INFO - Found 0 checkpoint files to load.\\n18:18:24 INFO - No examples are loaded from checkpoint files. 2 examples will be processed from scratch. Example IDs: {1, 2}.\\n18:18:24 INFO - Starting evaluation \u0026#x27;Gsm8k@40316ddb\u0026#x27; with 2 examples to evaluate.\\n18:18:28 INFO - Example 2 checkpointed to /tmp/test_run/run_20250122_6/Gsm8k/40316ddb/checkpoint.bagz.\\n18:18:29 INFO - Example 1 checkpointed to /tmp/test_run/run_20250122_6/Gsm8k/40316ddb/checkpoint.bagz.\\n18:18:29 INFO - 2 examples are checkpointed to /tmp/test_run/run_20250122_6/Gsm8k/40316ddb/checkpoint.bagz.\\n18:18:29 INFO - Gsm8k@40316ddb completed with 2 examples evaluated (0 from checkpoint, 2 newly processed).\\n18:18:29 INFO - Updated \u0026#x27;/tmp/test_run/run_20250122_6/Gsm8k/40316ddb/index.html\u0026#x27; in 0.26 seconds.\\n18:18:29 INFO - \u0026#x27;2.html\u0026#x27; generated in 1.27 seconds. \\n18:18:30 INFO - \u0026#x27;1.html\u0026#x27; generated in 1.28 seconds. \u003c/textarea\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003ciframe class=\"example-view\" name=\"example-view\" src=\"./1.html\" title=\"Example view.\"\u003e\u003c/iframe\u003e\u003c/div\u003e', style_files=StyleFiles(), styles=Styles(parts={'\\n        .label {\\n          display: inline-block;\\n          color: inherit;\\n          padding: 5px;\\n        }\\n        .label-container {\\n            display: inline-block;\\n        }\\n        ': 23, '\\n        .label-group {\\n          display: inline-flex;\\n          overflow: hidden;\\n          border-radius: 5px;\\n          border: 1px solid #DDD;\\n          padding: 0px;\\n          margin: 5px\\n        }\\n        .label-group .group-name {\\n          padding: 5px;\\n          color: white;\\n          background-color: dodgerblue;\\n        }\\n        .label-group .group-name \u003e .text {\\n          color: white;\\n        }\\n        .label-group .group-value {\\n        }\\n        .label-group .badge {\\n          border-radius: 0px;\\n        }\\n        ': 3, '\\n        span.tooltip {\\n          visibility: hidden;\\n          white-space: pre-wrap;\\n          font-weight: normal;\\n          background-color: #484848;\\n          color: #fff;\\n          padding: 10px;\\n          border-radius: 6px;\\n          position: absolute;\\n          z-index: 1;\\n        }\\n        span.tooltip:hover {\\n          visibility: visible;\\n        }\\n        .tooltip.html-content {\\n          white-space: inherit;\\n          background-color: white;\\n          color: inherit;\\n          box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\\n        }\\n        ': 9, '\\n        .label:hover + .tooltip {\\n          visibility: visible;\\n        }\\n        ': 9, '\\n        .progress-bar {\\n            display: inline-block;\\n        }\\n        .progress-bar .shade {\\n          position: relative;\\n          background-color: #EEE;\\n          display: inline-flex;\\n          margin: 10px;\\n          width: 100px; \\n          height: 5px;\\n          border-radius: 5px; \\n          overflow: hidden;\\n        }\\n        .progress-bar .sub-progress {\\n          background-color: dodgerblue;\\n          height: 10px;\\n          width: 0%;\\n          border-radius: 0px;\\n        }\\n        ': 1, '\\n        .eval-progress {\\n          display: inline-block;\\n        }\\n        .sub-progress.skipped {\\n          background-color:yellow;\\n        }\\n        .sub-progress.processed {\\n          background-color:#00B000;\\n        }\\n        .sub-progress.failed {\\n          background-color:red;\\n        }\\n        .progress-time {\\n          font-weight: normal;\\n          margin-left: 10px;\\n          border-radius: 5px;\\n          color: #CCC;\\n          padding: 5px;\\n        }\\n        ': 1, '\\n        .badge {\\n          background-color: #EEE;\\n          border-radius: 5px;\\n          color: #777;\\n        }\\n        ': 2, \"\\n        .usage-summary.label {\\n            display: inline-flex;\\n            border-radius: 5px;\\n            padding: 5px;\\n            background-color: #f1f1f1;\\n            color: #CCC;\\n        }\\n        .usage-summary.label::before {\\n            content: '$';\\n        }\\n        \": 1, '\\n        .metric-container {\\n          display: inline-flex;\\n          overflow: hidden;\\n          border-radius: 5px;\\n          border: 0px;\\n          margin: 5px;\\n          padding: 0px;\\n        }\\n        .metric-container .label-container {\\n          vertical-align: middle;\\n        }\\n        .metric-value.oop_errors {\\n          color: magenta;\\n          background-color: #f9e6eb;\\n        }\\n        .metric-value.non_oop_errors {\\n          color: red;\\n          background-color: #fdcccc;\\n        }\\n        ': 1, '\\n        .metric-name.match {\\n          padding: 5px;\\n          color: white;\\n          background-color: purple;\\n        }\\n        .metric-value.matches {\\n          color: green;\\n          background-color: #dcefbe;\\n        }\\n        .metric-value.mismatches {\\n          color: orange;\\n          background-color: #ffefc4;\\n        }\\n        ': 1, '\\n        .tab-control {\\n          border-spacing: 0px;\\n          border-collapse: collapse;\\n          margin-top: 10px;\\n        }\\n        .tab-control td {\\n          padding: 0px;\\n          margin: 0px;\\n          vertical-align: top;\\n        }\\n        .top.tab-button-group {\\n          border-left: 1px solid #DDD;\\n          border-top: 1px solid #DDD;\\n          border-right: 1px solid #DDD;\\n          border-radius: 5px 5px 0px 0px;\\n          padding: 0px 5px 0px 0px;\\n          margin-bottom: -2px;\\n        }\\n        .tab-button {\\n          background-color: transparent;\\n          border-radius: 5px;\\n          border: 0px;\\n          font-weight: bold;\\n          color: gray;\\n          outline: none;\\n          cursor: pointer;\\n          transition: 0.3s;\\n        }\\n        .tab-button:hover {\\n          background-color: #fff1dd;\\n        }\\n        .tab-button.selected {\\n          background-color: #f0ecf9;\\n          color: black;\\n        }\\n        .top.tab-content-group {\\n          border-left: 1px solid #DDD;\\n          border-right: 1px solid #DDD;\\n          border-bottom: 1px solid #DDD;\\n          border-radius: 0px 0px 5px 5px;\\n          margin: 0px;\\n          padding: 5px;\\n          height: 100%;\\n        }\\n        .top \u003e .tab-button {\\n          margin: 5px 0px 5px 5px;\\n        }\\n        .tab-content {\\n          display: none;\\n        }\\n        .tab-content.selected {\\n          display: block;\\n        }\\n        .left.tab-button-group {\\n          display: inline-flex;\\n          flex-direction: column;\\n          border: 1px solid #DDD;\\n          border-radius: 5px;\\n          margin-right: 5px;\\n          padding: 0px 0px 5px 0px;\\n        }\\n        .left.tab-content-group {\\n          border: 0px\\n          margin: 0px;\\n          padding: 0px;\\n          height: 100%;\\n        }\\n        .left \u003e .tab-button {\\n          text-align: left;\\n          margin: 5px 5px 0px 5px;\\n        }\\n        ': 2, '\\n        .experiment-summary {\\n            display: inline-block;\\n            font-weight: normal;\\n        }\\n        .experiment-name {\\n            font-weight: bold;\\n        }\\n        .experiment-dir.label {\\n            color: revert;\\n            margin-left: 0px;\\n            padding: 2px;\\n        }\\n        .usage-summary-badge {\\n          margin-left: 10px;\\n        }\\n        body {\\n          font: normal 16px \"Roboto\",\"Noto\",sans-serif;\\n        }\\n        ': 1, '\\n        details.eval-card {\\n          display: inline-block;\\n          border: 0px;\\n          box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\\n          margin: 15px;\\n        }\\n        .eval-card details {\\n          border: 0px;\\n        }\\n        .badge-groups {\\n          font-weight: normal;\\n          padding: 5px;\\n        }\\n        .parameter-group {\\n          display: inline-grid;\\n          grid-template-rows: auto auto;\\n          border: 0px;\\n          margin-right: 10px;\\n        }\\n        .parameter.badge {\\n          margin: 2px;\\n        }\\n        .metric-group {\\n          display: inline-grid;\\n          grid-template-rows: auto auto;\\n        }\\n        .eval-details .progress-bar \u003e .shade {\\n          visibility: hidden;\\n          width: 0px;\\n          margin: 0px;\\n        }\\n        .eval-details .progress-label {\\n          font-size: 16px;\\n          background-color: #eee;\\n        }\\n        .eval-details .progress-time {\\n          font-size: 16px;\\n          color: dodgerblue;\\n          background-color: #eee;\\n          margin-right: 10px;\\n        }\\n        .eval-details .usage-summary.badge {\\n          color: orange;\\n          font-size: 16px;\\n          background-color: #eee;\\n        }\\n        .eval-details .experiment-links {\\n          display: block;\\n          border: 0px;\\n          margin: 0px;\\n        }\\n        .eval-details .tab-control {\\n          width: 100%;\\n        }\\n        .eval-details .tab-button {\\n          font-size: large;\\n        }\\n        .experiment-links .label {\\n          color: revert;\\n          margin: 0px;\\n          padding: 2px;\\n        }\\n        .eval-details .experiment-id {\\n          font-size: 2.0em;\\n          font-weight: bold;\\n          display: block;\\n        }\\n        .eval-details .parameter-group {\\n          display: inline-block;\\n          padding: 5px;\\n        }\\n        .eval-definition {\\n          white-space: pre;\\n          background-color: #eee;\\n          padding: 15px;\\n        }\\n        .eval-details .metric-container {\\n          display: block;\\n          padding: 15px 0px;\\n        }\\n        .example-link {\\n          color: revert;\\n        }\\n        .example-view {\\n          border: 0px;\\n          width:100%;\\n          height:100%;\\n        }\\n        .logs-textarea {\\n          width: 100%;\\n          height: 500px;\\n          padding: 5px;\\n          border: 1px solid #DDD;\\n          background-color: #EEE;\\n          resize: vertical;\\n        }\\n        ': 1}), script_files=ScriptFiles(), scripts=Scripts(parts={\"\\n        function openTab(event, controlId, tabId) {\\n          const tabButtons = document.querySelectorAll('#' + controlId + '-button-group \u003e .tab-button');\\n          for (let i = 0; i \u003c tabButtons.length; i++) {\\n            tabButtons[i].classList.remove('selected');\\n          }\\n          const tabContents = document.querySelectorAll('#' + controlId + '-content-group \u003e .tab-content');\\n          for (let i = 0; i \u003c tabContents.length; i++) {\\n            tabContents[i].classList.remove('selected')\\n          }\\n          const tabButton = event.currentTarget;\\n          tabButton.classList.add('selected');\\n          document.getElementById(tabId).classList.add('selected');\\n        }\\n        \": 2}))"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pg.view(\n",
        "    benchmark.leaf_nodes[0],\n",
        "    extra_flags=dict(\n",
        "        current_run=run,\n",
        "        interactive=False,\n",
        "        card_view=False,\n",
        "    ),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-tWcFpMKPd-i"
      },
      "source": [
        "# View an detailed example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 762
        },
        "id": "WY2Jj1KOK-2A",
        "outputId": "87dae4f1-602d-41f1-8bdf-a8fbc5430117"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\u003chtml\u003e\n",
              "\u003chead\u003e\n",
              "\u003cstyle\u003e\n",
              ".label {\n",
              "  display: inline-block;\n",
              "  color: inherit;\n",
              "  padding: 5px;\n",
              "}\n",
              ".label-container {\n",
              "    display: inline-block;\n",
              "}\n",
              "span.tooltip {\n",
              "  visibility: hidden;\n",
              "  white-space: pre-wrap;\n",
              "  font-weight: normal;\n",
              "  background-color: #484848;\n",
              "  color: #fff;\n",
              "  padding: 10px;\n",
              "  border-radius: 6px;\n",
              "  position: absolute;\n",
              "  z-index: 1;\n",
              "}\n",
              "span.tooltip:hover {\n",
              "  visibility: visible;\n",
              "}\n",
              ".tooltip.html-content {\n",
              "  white-space: inherit;\n",
              "  background-color: white;\n",
              "  color: inherit;\n",
              "  box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\n",
              "}\n",
              ".label:hover + .tooltip {\n",
              "  visibility: visible;\n",
              "}\n",
              ".badge {\n",
              "  background-color: #EEE;\n",
              "  border-radius: 5px;\n",
              "  color: #777;\n",
              "}\n",
              ".usage-summary.label {\n",
              "    display: inline-flex;\n",
              "    border-radius: 5px;\n",
              "    padding: 5px;\n",
              "    background-color: #f1f1f1;\n",
              "    color: #CCC;\n",
              "}\n",
              ".usage-summary.label::before {\n",
              "    content: '$';\n",
              "}\n",
              ".label-group {\n",
              "  display: inline-flex;\n",
              "  overflow: hidden;\n",
              "  border-radius: 5px;\n",
              "  border: 1px solid #DDD;\n",
              "  padding: 0px;\n",
              "  margin: 5px\n",
              "}\n",
              ".label-group .group-name {\n",
              "  padding: 5px;\n",
              "  color: white;\n",
              "  background-color: dodgerblue;\n",
              "}\n",
              ".label-group .group-name \u003e .text {\n",
              "  color: white;\n",
              "}\n",
              ".label-group .group-value {\n",
              "}\n",
              ".label-group .badge {\n",
              "  border-radius: 0px;\n",
              "}\n",
              "/* Tooltip styles. */\n",
              "span.tooltip {\n",
              "  visibility: hidden;\n",
              "  white-space: pre-wrap;\n",
              "  font-weight: normal;\n",
              "  background-color: #484848;\n",
              "  color: #fff;\n",
              "  padding: 10px;\n",
              "  border-radius: 6px;\n",
              "  position: absolute;\n",
              "  z-index: 1;\n",
              "}\n",
              "span.tooltip:hover {\n",
              "  visibility: visible;\n",
              "}\n",
              "/* Summary styles. */\n",
              "details.pyglove summary {\n",
              "  font-weight: bold;\n",
              "  margin: -0.5em -0.5em 0;\n",
              "  padding: 0.5em;\n",
              "}\n",
              ".summary-name {\n",
              "  display: inline;\n",
              "  padding: 3px 5px 3px 5px;\n",
              "  margin: 0 5px;\n",
              "  border-radius: 3px;\n",
              "}\n",
              ".summary-title {\n",
              "  display: inline;\n",
              "}\n",
              ".summary-name + div.summary-title {\n",
              "  display: inline;\n",
              "  color: #aaa;\n",
              "}\n",
              ".summary-title:hover + span.tooltip {\n",
              "  visibility: visible;\n",
              "}\n",
              ".summary-name:hover \u003e span.tooltip {\n",
              "  visibility: visible;\n",
              "  background-color: darkblue;\n",
              "}\n",
              "/* Simple value styles. */\n",
              ".simple-value {\n",
              "  color: blue;\n",
              "  display: inline-block;\n",
              "  white-space: pre-wrap;\n",
              "  padding: 0.2em;\n",
              "  margin-top: 0.15em;\n",
              "}\n",
              ".simple-value.str {\n",
              "  color: darkred;\n",
              "  font-style: italic;\n",
              "}\n",
              ".simple-value.int, .simple-value.float {\n",
              "  color: darkblue;\n",
              "}\n",
              "/* Value details styles. */\n",
              "details.pyglove {\n",
              "  border: 1px solid #aaa;\n",
              "  border-radius: 4px;\n",
              "  padding: 0.5em 0.5em 0;\n",
              "  margin: 0.25em 0;\n",
              "}\n",
              "details.pyglove[open] {\n",
              "  padding: 0.5em 0.5em 0.5em;\n",
              "}\n",
              ".highlight {\n",
              "  background-color: Mark;\n",
              "}\n",
              ".lowlight {\n",
              "  opacity: 0.2;\n",
              "}\n",
              "/* Complex value styles. */\n",
              "span.empty-container::before {\n",
              "    content: '(empty)';\n",
              "    font-style: italic;\n",
              "    margin-left: 0.5em;\n",
              "    color: #aaa;\n",
              "}\n",
              "/* Object key styles. */\n",
              ".object-key {\n",
              "  margin: 0.15em 0.3em 0.15em 0;\n",
              "  display: block;\n",
              "}\n",
              ".object-key:hover + .tooltip {\n",
              "  visibility: visible;\n",
              "  background-color: darkblue;\n",
              "}\n",
              ".object-key.str {\n",
              "  color: gray;\n",
              "  border: 1px solid lightgray;\n",
              "  background-color: ButtonFace;\n",
              "  border-radius: 0.2em;\n",
              "  padding: 0.3em;\n",
              "}\n",
              ".object-key.int::before{\n",
              "  content: '[';\n",
              "}\n",
              ".object-key.int::after{\n",
              "  content: ']';\n",
              "}\n",
              ".object-key.int{\n",
              "  border: 0;\n",
              "  color: lightgray;\n",
              "  background-color: transparent;\n",
              "  border-radius: 0;\n",
              "  padding: 0;\n",
              "}\n",
              ".tab-control {\n",
              "  border-spacing: 0px;\n",
              "  border-collapse: collapse;\n",
              "  margin-top: 10px;\n",
              "}\n",
              ".tab-control td {\n",
              "  padding: 0px;\n",
              "  margin: 0px;\n",
              "  vertical-align: top;\n",
              "}\n",
              ".top.tab-button-group {\n",
              "  border-left: 1px solid #DDD;\n",
              "  border-top: 1px solid #DDD;\n",
              "  border-right: 1px solid #DDD;\n",
              "  border-radius: 5px 5px 0px 0px;\n",
              "  padding: 0px 5px 0px 0px;\n",
              "  margin-bottom: -2px;\n",
              "}\n",
              ".tab-button {\n",
              "  background-color: transparent;\n",
              "  border-radius: 5px;\n",
              "  border: 0px;\n",
              "  font-weight: bold;\n",
              "  color: gray;\n",
              "  outline: none;\n",
              "  cursor: pointer;\n",
              "  transition: 0.3s;\n",
              "}\n",
              ".tab-button:hover {\n",
              "  background-color: #fff1dd;\n",
              "}\n",
              ".tab-button.selected {\n",
              "  background-color: #f0ecf9;\n",
              "  color: black;\n",
              "}\n",
              ".top.tab-content-group {\n",
              "  border-left: 1px solid #DDD;\n",
              "  border-right: 1px solid #DDD;\n",
              "  border-bottom: 1px solid #DDD;\n",
              "  border-radius: 0px 0px 5px 5px;\n",
              "  margin: 0px;\n",
              "  padding: 5px;\n",
              "  height: 100%;\n",
              "}\n",
              ".top \u003e .tab-button {\n",
              "  margin: 5px 0px 5px 5px;\n",
              "}\n",
              ".tab-content {\n",
              "  display: none;\n",
              "}\n",
              ".tab-content.selected {\n",
              "  display: block;\n",
              "}\n",
              ".left.tab-button-group {\n",
              "  display: inline-flex;\n",
              "  flex-direction: column;\n",
              "  border: 1px solid #DDD;\n",
              "  border-radius: 5px;\n",
              "  margin-right: 5px;\n",
              "  padding: 0px 0px 5px 0px;\n",
              "}\n",
              ".left.tab-content-group {\n",
              "  border: 0px\n",
              "  margin: 0px;\n",
              "  padding: 0px;\n",
              "  height: 100%;\n",
              "}\n",
              ".left \u003e .tab-button {\n",
              "  text-align: left;\n",
              "  margin: 5px 5px 0px 5px;\n",
              "}\n",
              "/* Langfun Template styles. */\n",
              ".template-str {\n",
              "    padding: 10px;\n",
              "    margin: 10px 5px 10px 5px;\n",
              "    font-style: italic;\n",
              "    font-size: 1.1em;\n",
              "    white-space: pre-wrap;\n",
              "    border: 1px solid #EEE;\n",
              "    border-radius: 5px;\n",
              "    background-color: #EEE;\n",
              "    color: #cc2986;\n",
              "}\n",
              ".lf-schema-definition {\n",
              "    color: blue;\n",
              "    margin: 5px;\n",
              "    white-space: pre-wrap;\n",
              "}\n",
              "/* Langfun Message styles.*/\n",
              "[class^=\"message-\"] \u003e details {\n",
              "    margin: 0px 0px 5px 0px;\n",
              "    border: 1px solid #EEE;\n",
              "}\n",
              ".lf-message.summary-title::after {\n",
              "    content: ' 💬';\n",
              "}\n",
              "details.pyglove.ai-message {\n",
              "    border: 1px solid blue;\n",
              "    color: blue;\n",
              "}\n",
              "details.pyglove.user-message {\n",
              "    border: 1px solid green;\n",
              "    color: green;\n",
              "}\n",
              ".message-tags {\n",
              "    margin: 5px 0px 5px 0px;\n",
              "    font-size: .8em;\n",
              "}\n",
              ".message-tags \u003e span {\n",
              "    border-radius: 5px;\n",
              "    background-color: #CCC;\n",
              "    padding: 3px;\n",
              "    margin: 0px 2px 0px 2px;\n",
              "    color: white;\n",
              "}\n",
              ".message-text {\n",
              "    padding: 20px;\n",
              "    margin: 10px 5px 10px 5px;\n",
              "    font-style: italic;\n",
              "    white-space: pre-wrap;\n",
              "    border: 1px solid #EEE;\n",
              "    border-radius: 5px;\n",
              "    background-color: #EEE;\n",
              "}\n",
              ".modality-in-text {\n",
              "    display: inline-block;\n",
              "}\n",
              ".modality-in-text \u003e details.pyglove {\n",
              "    display: inline-block;\n",
              "    font-size: 0.8em;\n",
              "    border: 0;\n",
              "    background-color: #A6F1A6;\n",
              "    margin: 0px 5px 0px 5px;\n",
              "}\n",
              ".message-result {\n",
              "    color: dodgerblue;\n",
              "}\n",
              ".message-usage {\n",
              "    color: orange;\n",
              "}\n",
              ".message-usage .object-key.str {\n",
              "    border: 1px solid orange;\n",
              "    background-color: orange;\n",
              "    color: white;\n",
              "}\n",
              ".query-invocation-title {\n",
              "  display: inline-block;\n",
              "  font-weight: normal;\n",
              "}\n",
              ".query-invocation-type-name {\n",
              "  color: #888;\n",
              "}\n",
              ".query-invocation-lm.badge {\n",
              "  margin-left: 5px;\n",
              "  margin-right: 5px;\n",
              "  color: white;\n",
              "  background-color: mediumslateblue;\n",
              "}\n",
              ".query-invocation-time.badge {\n",
              "  margin-left: 5px;\n",
              "  border-radius: 0px;\n",
              "  font-weight: bold;\n",
              "  background-color: aliceblue;\n",
              "}\n",
              ".query-invocation-title .usage-summary.label {\n",
              "  border-radius: 0px;\n",
              "  color: #AAA;\n",
              "}\n",
              ".example-container {\n",
              "  display: block;\n",
              "  padding: 10px;\n",
              "}\n",
              ".example-id {\n",
              "  font-weight: bold;\n",
              "  font-size: 40px;\n",
              "  margin: 0 10px;\n",
              "  vertical-align: middle;\n",
              "}\n",
              "a.previous, a.next {\n",
              "  text-decoration: none;\n",
              "  vertical-align: middle;\n",
              "  display: inline-block;\n",
              "  padding: 8px 8px;\n",
              "  color: #DDD;\n",
              "}\n",
              "a.previous:hover, a.next:hover {\n",
              "  background-color: #ddd;\n",
              "  color: black;\n",
              "}\n",
              "/* Badge styles. */\n",
              ".eval-example .badge.match {\n",
              "  color: green;\n",
              "  background-color: #dcefbe;\n",
              "}\n",
              ".eval-example .badge.error {\n",
              "  color: red;\n",
              "  background-color: #fdcccc;\n",
              "}\n",
              ".eval-example .badge.mismatch {\n",
              "  color: orange;\n",
              "  background-color: #ffefc4;\n",
              "}\n",
              ".eval-example .badge.score {\n",
              "  color: blue;\n",
              "  background-color: #c4dced;\n",
              "}\n",
              "\u003c/style\u003e\n",
              "\u003cscript\u003e\n",
              "function openTab(event, controlId, tabId) {\n",
              "  const tabButtons = document.querySelectorAll('#' + controlId + '-button-group \u003e .tab-button');\n",
              "  for (let i = 0; i \u003c tabButtons.length; i++) {\n",
              "    tabButtons[i].classList.remove('selected');\n",
              "  }\n",
              "  const tabContents = document.querySelectorAll('#' + controlId + '-content-group \u003e .tab-content');\n",
              "  for (let i = 0; i \u003c tabContents.length; i++) {\n",
              "    tabContents[i].classList.remove('selected')\n",
              "  }\n",
              "  const tabButton = event.currentTarget;\n",
              "  tabButton.classList.add('selected');\n",
              "  document.getElementById(tabId).classList.add('selected');\n",
              "}\n",
              "\u003c/script\u003e\n",
              "\u003c/head\u003e\n",
              "\u003cbody\u003e\n",
              "\u003cdiv class=\"eval-example\"\u003e\u003cdiv\u003e\u003cdiv\u003e\u003cspan class=\"label example-id\"\u003e#1\u003c/span\u003e\u003ca class=\"label next\" href=\"2.html\"\u003e▶\u003c/a\u003e\u003c/div\u003e\u003cdiv class=\"example-container\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138085545196752\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085545128208\"\u003eUsageSummary(\n",
              "  cached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  ),\n",
              "  uncached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 180,\n",
              "      completion_tokens = 136,\n",
              "      total_tokens = 316,\n",
              "      num_requests = 1,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 2.882,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {\n",
              "      OpenAI(gpt-4o-mini) = LMSamplingUsage(\n",
              "        prompt_tokens = 180,\n",
              "        completion_tokens = 136,\n",
              "        total_tokens = 316,\n",
              "        num_requests = 1,\n",
              "        estimated_cost = 0.000,\n",
              "        retry_stats = RetryStats(\n",
              "          num_occurences = 0,\n",
              "          total_wait_interval = 0.000,\n",
              "          total_call_interval = 2.882,\n",
              "          errors = {}\n",
              "        )\n",
              "      )\n",
              "    }\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-group\"\u003e\u003cspan class=\"label match badge group-value\"\u003ematch\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085542756816-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085542756816', 'control-138085542756816-0')\"\u003e\u003cspan class=\"label\"\u003eInput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085542756816', 'control-138085542756816-1')\"\u003e\u003cspan class=\"label\"\u003eOutput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, 'control-138085542756816', 'control-138085542756816-2')\"\u003e\u003cspan class=\"label\"\u003eOutput Metadata\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085542756816-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085542756816-0\"\u003e\u003cdetails open class=\"pyglove dict\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eDict(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e{\n",
              "  \u0026#x27;question\u0026#x27;: \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day...\u0026quot;,\n",
              "  \u0026#x27;answer\u0026#x27;: 18\n",
              "}\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value dict\"\u003e\u003cdetails open class=\"pyglove str\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003equestion\u003cspan class=\"tooltip\"\u003einput.question\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003estr\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e\u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day...\u0026quot;\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value str\"\u003eJanet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u003c/span\u003e\u003c/details\u003e\u003cdetails open class=\"pyglove int\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003eanswer\u003cspan class=\"tooltip\"\u003einput.answer\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eint\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e18\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value int\"\u003e18\u003c/span\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085542756816-1\"\u003e\u003cdetails open class=\"pyglove answer\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eAnswer(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003eAnswer(\n",
              "  step_by_step_thoughts=[\n",
              "    \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\n",
              "    \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\n",
              "    \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\n",
              "    \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\n",
              "    \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\n",
              "    \u0026#x27;She sells each egg for $2.\u0026#x27;,\n",
              "    \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\n",
              "  ],\n",
              "  final_answer=18\n",
              ")\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value answer\"\u003e\u003cdetails open class=\"pyglove list\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003estep_by_step_thoughts\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eList(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e[\n",
              "  \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\n",
              "  \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\n",
              "  \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\n",
              "  \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\n",
              "  \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\n",
              "  \u0026#x27;She sells each egg for $2.\u0026#x27;,\n",
              "  \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\n",
              "]\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value list\"\u003e\u003ctable\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e0\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[0]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e1\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[1]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e2\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[2]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e3\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[3]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e4\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[4]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e5\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[5]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She sells each egg for $2.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e6\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[6]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/details\u003e\u003cdetails open class=\"pyglove int\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003efinal_answer\u003cspan class=\"tooltip\"\u003eoutput.final_answer\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eint\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e18\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value int\"\u003e18\u003c/span\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085542756816-2\"\u003e\u003cdetails open class=\"pyglove dict\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003edict(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e{\n",
              "  \u0026#x27;queries\u0026#x27;: [\n",
              "    QueryInvocation(\n",
              "      input=Ref(\n",
              "        value = Template(\n",
              "          template_str = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\n",
              "          clean = True\n",
              "        )\n",
              "      ),\n",
              "      schema=Schema(\n",
              "        spec=Object(\n",
              "          Answer\n",
              "        )\n",
              "      ),\n",
              "      lm_response=AIMessage(\n",
              "        text=\u0026#x27;```python\\nAnswer(\\n    step_by_step_thoughts=[\\n        \u0026quot;Janet\\\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\\n        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\\n        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\\n ...\u0026#x27;,\n",
              "        sender=\u0026#x27;AI\u0026#x27;,\n",
              "        metadata={},\n",
              "        tags=[]\n",
              "      ),\n",
              "      lm=Ref(\n",
              "        value = Gpt4oMini(\n",
              "          sampling_options = LMSamplingOptions(\n",
              "            temperature = None,\n",
              "            max_tokens = None,\n",
              "            n = 1,\n",
              "            top_k = 40,\n",
              "            top_p = None,\n",
              "            stop = None,\n",
              "            random_seed = None,\n",
              "            logprobs = False,\n",
              "            top_logprobs = None\n",
              "          ),\n",
              "          cache = ContextualAttribute(\n",
              "            type = None,\n",
              "            default = None\n",
              "          ),\n",
              "          max_concurrency = None,\n",
              "          timeout = 120.0,\n",
              "          max_attempts = 5,\n",
              "          retry_interval = (5, 60),\n",
              "          exponential_backoff = True,\n",
              "          max_retry_interval = 300,\n",
              "          debug = False,\n",
              "          api_endpoint = \u0026#x27;https://api.openai.com/v1/chat/completions\u0026#x27;,\n",
              "          model = \u0026#x27;gpt-4o-mini\u0026#x27;,\n",
              "          headers = None,\n",
              "          multimodal = True,\n",
              "          api_key = None,\n",
              "          organization = None,\n",
              "          project = None\n",
              "        )\n",
              "      ),\n",
              "      examples=[],\n",
              "      usage_summary=UsageSummary(\n",
              "        cached=AggregatedUsage(\n",
              "          total=LMSamplingUsage(\n",
              "            prompt_tokens=0,\n",
              "            completion_tokens=0,\n",
              "            total_tokens=0,\n",
              "            num_requests=0,\n",
              "            estimated_cost=0.0,\n",
              "            retry_stats=RetryStats(\n",
              "              num_occurences=0,\n",
              "              total_wait_interval=0.0,\n",
              "              total_call_interval=0.0,\n",
              "              errors={}\n",
              "            )\n",
              "          ),\n",
              "          breakdown={}\n",
              "        ),\n",
              "        uncached=AggregatedUsage(\n",
              "          total=LMSamplingUsage(\n",
              "            prompt_tokens=180,\n",
              "            completion_tokens=136,\n",
              "            total_tokens=316,\n",
              "            num_requests=1,\n",
              "            estimated_cost=0.00010859999999999998,\n",
              "            retry_stats=RetryStats(\n",
              "              num_occurences=0,\n",
              "              total_wait_interval=0.0,\n",
              "              total_call_interval=2.882235288619995,\n",
              "              errors={}\n",
              "            )\n",
              "          ),\n",
              "          breakdown={\n",
              "            \u0026#x27;OpenAI(gpt-4o-mini)\u0026#x27;: LMSamplingUsage(\n",
              "              prompt_tokens=180,\n",
              "              completion_tokens=136,\n",
              "              total_tokens=316,\n",
              "              num_requests=1,\n",
              "              estimated_cost=0.00010859999999999998,\n",
              "              retry_stats=RetryStats(\n",
              "                num_occurences=0,\n",
              "                total_wait_interval=0.0,\n",
              "                total_call_interval=2.882235288619995,\n",
              "                errors={}\n",
              "              )\n",
              "            )\n",
              "          }\n",
              "        )\n",
              "      ),\n",
              "      start_time=1737569905.6691015,\n",
              "      end_time=1737569908.6945176\n",
              "    )\n",
              "  ]\n",
              "}\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value dict\"\u003e\u003cdetails open class=\"pyglove list\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003equeries\u003cspan class=\"tooltip\"\u003emetadata.queries\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003elist(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e[\n",
              "  QueryInvocation(\n",
              "    input=Ref(\n",
              "      value = Template(\n",
              "        template_str = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\n",
              "        clean = True\n",
              "      )\n",
              "    ),\n",
              "    schema=Schema(\n",
              "      spec=Object(\n",
              "        Answer\n",
              "      )\n",
              "    ),\n",
              "    lm_response=AIMessage(\n",
              "      text=\u0026#x27;```python\\nAnswer(\\n    step_by_step_thoughts=[\\n        \u0026quot;Janet\\\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\\n        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\\n        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\\n ...\u0026#x27;,\n",
              "      sender=\u0026#x27;AI\u0026#x27;,\n",
              "      metadata={},\n",
              "      tags=[]\n",
              "    ),\n",
              "    lm=Ref(\n",
              "      value = Gpt4oMini(\n",
              "        sampling_options = LMSamplingOptions(\n",
              "          temperature = None,\n",
              "          max_tokens = None,\n",
              "          n = 1,\n",
              "          top_k = 40,\n",
              "          top_p = None,\n",
              "          stop = None,\n",
              "          random_seed = None,\n",
              "          logprobs = False,\n",
              "          top_logprobs = None\n",
              "        ),\n",
              "        cache = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = None\n",
              "        ),\n",
              "        max_concurrency = None,\n",
              "        timeout = 120.0,\n",
              "        max_attempts = 5,\n",
              "        retry_interval = (5, 60),\n",
              "        exponential_backoff = True,\n",
              "        max_retry_interval = 300,\n",
              "        debug = False,\n",
              "        api_endpoint = \u0026#x27;https://api.openai.com/v1/chat/completions\u0026#x27;,\n",
              "        model = \u0026#x27;gpt-4o-mini\u0026#x27;,\n",
              "        headers = None,\n",
              "        multimodal = True,\n",
              "        api_key = None,\n",
              "        organization = None,\n",
              "        project = None\n",
              "      )\n",
              "    ),\n",
              "    examples=[],\n",
              "    usage_summary=UsageSummary(\n",
              "      cached=AggregatedUsage(\n",
              "        total=LMSamplingUsage(\n",
              "          prompt_tokens=0,\n",
              "          completion_tokens=0,\n",
              "          total_tokens=0,\n",
              "          num_requests=0,\n",
              "          estimated_cost=0.0,\n",
              "          retry_stats=RetryStats(\n",
              "            num_occurences=0,\n",
              "            total_wait_interval=0.0,\n",
              "            total_call_interval=0.0,\n",
              "            errors={}\n",
              "          )\n",
              "        ),\n",
              "        breakdown={}\n",
              "      ),\n",
              "      uncached=AggregatedUsage(\n",
              "        total=LMSamplingUsage(\n",
              "          prompt_tokens=180,\n",
              "          completion_tokens=136,\n",
              "          total_tokens=316,\n",
              "          num_requests=1,\n",
              "          estimated_cost=0.00010859999999999998,\n",
              "          retry_stats=RetryStats(\n",
              "            num_occurences=0,\n",
              "            total_wait_interval=0.0,\n",
              "            total_call_interval=2.882235288619995,\n",
              "            errors={}\n",
              "          )\n",
              "        ),\n",
              "        breakdown={\n",
              "          \u0026#x27;OpenAI(gpt-4o-mini)\u0026#x27;: LMSamplingUsage(\n",
              "            prompt_tokens=180,\n",
              "            completion_tokens=136,\n",
              "            total_tokens=316,\n",
              "            num_requests=1,\n",
              "            estimated_cost=0.00010859999999999998,\n",
              "            retry_stats=RetryStats(\n",
              "              num_occurences=0,\n",
              "              total_wait_interval=0.0,\n",
              "              total_call_interval=2.882235288619995,\n",
              "              errors={}\n",
              "            )\n",
              "          )\n",
              "        }\n",
              "      )\n",
              "    ),\n",
              "    start_time=1737569905.6691015,\n",
              "    end_time=1737569908.6945176\n",
              "  )\n",
              "]\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value list\"\u003e\u003ctable\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e0\u003c/span\u003e\u003cspan class=\"tooltip\"\u003emetadata.queries[0]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cdetails open class=\"pyglove query-invocation\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003e\u003cdiv class=\"query-invocation-title\"\u003e\u003cspan class=\"label query-invocation-type-name\"\u003elf.query\u003c/span\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label query-invocation-lm badge\"\u003elm=OpenAI(gpt-4o-mini)\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eGpt4oMini()\u003c/span\u003e\u003c/div\u003e\u003cspan class=\"label query-invocation-time badge\"\u003e3 seconds\u003c/span\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138085542613200\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085545041744\"\u003eUsageSummary(\n",
              "  cached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 0,\n",
              "      completion_tokens = 0,\n",
              "      total_tokens = 0,\n",
              "      num_requests = 0,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 0.000,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {}\n",
              "  ),\n",
              "  uncached = AggregatedUsage(\n",
              "    total = LMSamplingUsage(\n",
              "      prompt_tokens = 180,\n",
              "      completion_tokens = 136,\n",
              "      total_tokens = 316,\n",
              "      num_requests = 1,\n",
              "      estimated_cost = 0.000,\n",
              "      retry_stats = RetryStats(\n",
              "        num_occurences = 0,\n",
              "        total_wait_interval = 0.000,\n",
              "        total_call_interval = 2.882,\n",
              "        errors = {}\n",
              "      )\n",
              "    ),\n",
              "    breakdown = {\n",
              "      OpenAI(gpt-4o-mini) = LMSamplingUsage(\n",
              "        prompt_tokens = 180,\n",
              "        completion_tokens = 136,\n",
              "        total_tokens = 316,\n",
              "        num_requests = 1,\n",
              "        estimated_cost = 0.000,\n",
              "        retry_stats = RetryStats(\n",
              "          num_occurences = 0,\n",
              "          total_wait_interval = 0.000,\n",
              "          total_call_interval = 2.882,\n",
              "          errors = {}\n",
              "        )\n",
              "      )\n",
              "    }\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/summary\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085516702992-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085516702992', 'control-138085516702992-0')\"\u003e\u003cspan class=\"label\"\u003einput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, 'control-138085516702992', 'control-138085516702992-1')\"\u003e\u003cspan class=\"label\"\u003eoutput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085516702992', 'control-138085516702992-2')\"\u003e\u003cspan class=\"label\"\u003eschema\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085516702992', 'control-138085516702992-3')\"\u003e\u003cspan class=\"label\"\u003elm_request\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085516702992', 'control-138085516702992-4')\"\u003e\u003cspan class=\"label\"\u003elm_response\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085516702992-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-0\"\u003e\u003cdetails open class=\"pyglove template lf-template\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title lf-template\"\u003eTemplate(...)\u003c/div\u003e\u003cspan class=\"tooltip lf-template\"\u003eTemplate(\n",
              "  template_str=\u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day...\u0026quot;,\n",
              "  clean=True\n",
              ")\u003c/span\u003e\u003c/summary\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085516558480-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, 'control-138085516558480', 'control-138085516558480-0')\"\u003e\u003cspan class=\"label\"\u003etemplate_str\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, 'control-138085516558480', 'control-138085516558480-1')\"\u003e\u003cspan class=\"label\"\u003evariables\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085516558480-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516558480-0\"\u003e\u003cdiv class=\"template-str\"\u003e\u003cspan\u003eJanet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085516558480-1\"\u003e\u003cdiv class=\"complex-value template\"\u003e\u003cspan class=\"empty-container\"\u003e\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085516702992-1\"\u003e\u003cdetails open class=\"pyglove answer\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eAnswer(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003eAnswer(\n",
              "  step_by_step_thoughts=[\n",
              "    \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\n",
              "    \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\n",
              "    \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\n",
              "    \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\n",
              "    \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\n",
              "    \u0026#x27;She sells each egg for $2.\u0026#x27;,\n",
              "    \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\n",
              "  ],\n",
              "  final_answer=18\n",
              ")\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value answer\"\u003e\u003cdetails open class=\"pyglove list\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003estep_by_step_thoughts\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eList(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e[\n",
              "  \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\n",
              "  \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\n",
              "  \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\n",
              "  \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\n",
              "  \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\n",
              "  \u0026#x27;She sells each egg for $2.\u0026#x27;,\n",
              "  \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\n",
              "]\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value list\"\u003e\u003ctable\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e0\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[0]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e1\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[1]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e2\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[2]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e3\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[3]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e4\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[4]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e5\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[5]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She sells each egg for $2.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e6\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[6]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/details\u003e\u003cdetails open class=\"pyglove int\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003efinal_answer\u003cspan class=\"tooltip\"\u003efinal_answer\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eint\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e18\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value int\"\u003e18\u003c/span\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-2\"\u003e\u003cdetails open class=\"pyglove schema\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eSchema(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003eSchema(\n",
              "  spec=Object(\n",
              "    Answer\n",
              "  )\n",
              ")\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"lf-schema-definition\"\u003eAnswer\n",
              "\n",
              "```python\n",
              "class Answer:\n",
              "  step_by_step_thoughts: list[str]\n",
              "  final_answer: int\n",
              "```\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-3\"\u003e\u003cdetails open class=\"pyglove user-message lf-message\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title lf-message\"\u003eUserMessage(...)\u003c/div\u003e\u003cspan class=\"tooltip lf-message\"\u003eUserMessage(\n",
              "  text=\u0026#x27;Please respond to the last INPUT_OBJECT with OUTPUT_OBJECT according to OUTPUT_TYPE.\\n\\nINPUT_OBJECT:\\n  1 + 1 =\\n\\nOUTPUT_TYPE:\\n  Answer\\n\\n  ```python\\n  class Answer:\\n    final_answer: int\\n  ```\\n\\nOUTPUT_OBJECT:\\n  ```python\\n  Answer(\\n    final_answer=2\\n  )\\n  ```...\u0026#x27;,\n",
              "  sender=\u0026#x27;User\u0026#x27;,\n",
              "  metadata={\n",
              "    \u0026#x27;mapping_template\u0026#x27;: Ref(\n",
              "      value = Template(\n",
              "        template_str = \u0026#x27;{%- if example.context -%}\\n{{ context_title}}:\\n{{ example.context | indent(2, True)}}\\n\\n{% endif -%}\\n\\n{{ input_title }}:\\n{{ example.input_repr(protocol, compact=False) | indent(2, True) }}\\n\\n{% if example.schema -%}\\n{{ schema_title }}:\\n{{ example.schema_repr(protocol) | indent(2, True) }}\\n\\n{% endif -%}\\n\\n{{ output_title }}:\\n{%- if example.has_output %}\\n{{ example.output_repr(protocol, compact=False) | indent(2, True) }}\\n{% endif -%}\u0026#x27;,\n",
              "        clean = True,\n",
              "        context_title = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        input_title = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        output_title = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        schema_title = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        protocol = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        example = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        )\n",
              "      )\n",
              "    ),\n",
              "    \u0026#x27;examples\u0026#x27;: None,\n",
              "    \u0026#x27;preamble\u0026#x27;: Ref(\n",
              "      value = LangFunc(\n",
              "        template_str = \u0026#x27;Please respond to the last {{ input_title }} with {{ output_title }} according to {{ schema_title }}.\\n\\n{{ input_title }}:\\n  1 + 1 =\\n\\n{{ schema_title }}:\\n  Answer\\n\\n  ```python\\n  class Answer:\\n    final_answer: int\\n  ```\\n\\n{{ output_title }}:\\n  ```python\\n  Answer(\\n    final_answer=2\\n  )\\n  ```\u0026#x27;,\n",
              "        clean = True,\n",
              "        lm = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        input_title = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        output_title = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        ),\n",
              "        schema_title = ContextualAttribute(\n",
              "          type = None,\n",
              "          default = (MISSING_VALUE)\n",
              "        )\n",
              "      )\n",
              "    ),\n",
              "    \u0026#x27;mapping_request\u0026#x27;: Ref(\n",
              "      value = MappingExample(\n",
              "        input = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\n",
              "        output = MISSING(Any),\n",
              "        schema = Ref(\n",
              "          value = Schema(\n",
              "            spec = Object(\n",
              "              Answer\n",
              "            )\n",
              "          )\n",
              "        ),\n",
              "        context = None,\n",
              "        metadata = {}\n",
              "      )\n",
              "    ),\n",
              "    \u0026#x27;input\u0026#x27;: Ref(\n",
              "      value = UserMessage(\n",
              "        text = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\n",
              "        sender = \u0026#x27;User\u0026#x27;,\n",
              "        metadata = {},\n",
              "        tags = [\n",
              "          0 : \u0026#x27;rendered\u0026#x27;\n",
              "        ]\n",
              "      )\n",
              "    ),\n",
              "    \u0026#x27;schema\u0026#x27;: Ref(\n",
              "      value = Schema(\n",
              "        spec = Object(\n",
              "          Answer\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  },\n",
              "  tags=[\n",
              "    \u0026#x27;rendered\u0026#x27;\n",
              "  ]\n",
              ")\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex_value\"\u003e\u003cdiv class=\"message-tags\"\u003e\u003cspan\u003erendered\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"message-text\"\u003ePlease respond to the last INPUT_OBJECT with OUTPUT_OBJECT according to OUTPUT_TYPE.\n",
              "\n",
              "INPUT_OBJECT:\n",
              "  1 + 1 =\n",
              "\n",
              "OUTPUT_TYPE:\n",
              "  Answer\n",
              "\n",
              "  ```python\n",
              "  class Answer:\n",
              "    final_answer: int\n",
              "  ```\n",
              "\n",
              "OUTPUT_OBJECT:\n",
              "  ```python\n",
              "  Answer(\n",
              "    final_answer=2\n",
              "  )\n",
              "  ```\n",
              "\n",
              "INPUT_OBJECT:\n",
              "  Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\n",
              "\n",
              "OUTPUT_TYPE:\n",
              "  Answer\n",
              "\n",
              "  ```python\n",
              "  class Answer:\n",
              "    step_by_step_thoughts: list[str]\n",
              "    final_answer: int\n",
              "  ```\n",
              "\n",
              "OUTPUT_OBJECT:\u003c/div\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-4\"\u003e\u003cdetails open class=\"pyglove ai-message lf-message\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title lf-message\"\u003eAIMessage(...)\u003c/div\u003e\u003cspan class=\"tooltip lf-message\"\u003eAIMessage(\n",
              "  text=\u0026#x27;```python\\nAnswer(\\n    step_by_step_thoughts=[\\n        \u0026quot;Janet\\\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\\n        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\\n        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\\n ...\u0026#x27;,\n",
              "  sender=\u0026#x27;AI\u0026#x27;,\n",
              "  metadata={},\n",
              "  tags=[]\n",
              ")\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex_value\"\u003e\u003cdiv class=\"message-tags\"\u003e\u003c/div\u003e\u003cdiv class=\"message-text\"\u003e```python\n",
              "Answer(\n",
              "    step_by_step_thoughts=[\n",
              "        \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\n",
              "        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\n",
              "        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\n",
              "        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\n",
              "        \u0026quot;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026quot;,\n",
              "        \u0026quot;She sells each egg for $2.\u0026quot;,\n",
              "        \u0026quot;Total money made = 9 eggs * $2/egg = $18.\u0026quot;\n",
              "    ],\n",
              "    final_answer=18\n",
              ")\n",
              "```\u003c/div\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/details\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\n",
              "\u003c/body\u003e\n",
              "\u003c/html\u003e"
            ],
            "text/plain": [
              "Html(content='\u003cdiv class=\"eval-example\"\u003e\u003cdiv\u003e\u003cdiv\u003e\u003cspan class=\"label example-id\"\u003e#1\u003c/span\u003e\u003ca class=\"label next\" href=\"2.html\"\u003e▶\u003c/a\u003e\u003c/div\u003e\u003cdiv class=\"example-container\"\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138085545196752\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085545128208\"\u003eUsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 180,\\n      completion_tokens = 136,\\n      total_tokens = 316,\\n      num_requests = 1,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 2.882,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 180,\\n        completion_tokens = 136,\\n        total_tokens = 316,\\n        num_requests = 1,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 2.882,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"label-group\"\u003e\u003cspan class=\"label match badge group-value\"\u003ematch\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085542756816-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085542756816\\', \\'control-138085542756816-0\\')\"\u003e\u003cspan class=\"label\"\u003eInput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085542756816\\', \\'control-138085542756816-1\\')\"\u003e\u003cspan class=\"label\"\u003eOutput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, \\'control-138085542756816\\', \\'control-138085542756816-2\\')\"\u003e\u003cspan class=\"label\"\u003eOutput Metadata\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085542756816-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085542756816-0\"\u003e\u003cdetails open class=\"pyglove dict\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eDict(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e{\\n  \u0026#x27;question\u0026#x27;: \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day...\u0026quot;,\\n  \u0026#x27;answer\u0026#x27;: 18\\n}\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value dict\"\u003e\u003cdetails open class=\"pyglove str\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003equestion\u003cspan class=\"tooltip\"\u003einput.question\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003estr\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e\u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day...\u0026quot;\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value str\"\u003eJanet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u003c/span\u003e\u003c/details\u003e\u003cdetails open class=\"pyglove int\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003eanswer\u003cspan class=\"tooltip\"\u003einput.answer\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eint\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e18\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value int\"\u003e18\u003c/span\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085542756816-1\"\u003e\u003cdetails open class=\"pyglove answer\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eAnswer(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003eAnswer(\\n  step_by_step_thoughts=[\\n    \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n    \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\\n    \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\\n    \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\\n    \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\\n    \u0026#x27;She sells each egg for $2.\u0026#x27;,\\n    \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\\n  ],\\n  final_answer=18\\n)\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value answer\"\u003e\u003cdetails open class=\"pyglove list\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003estep_by_step_thoughts\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eList(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e[\\n  \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n  \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\\n  \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\\n  \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\\n  \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\\n  \u0026#x27;She sells each egg for $2.\u0026#x27;,\\n  \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\\n]\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value list\"\u003e\u003ctable\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e0\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[0]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e1\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[1]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e2\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[2]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e3\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[3]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e4\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[4]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e5\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[5]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She sells each egg for $2.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e6\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eoutput.step_by_step_thoughts[6]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/details\u003e\u003cdetails open class=\"pyglove int\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003efinal_answer\u003cspan class=\"tooltip\"\u003eoutput.final_answer\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eint\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e18\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value int\"\u003e18\u003c/span\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085542756816-2\"\u003e\u003cdetails open class=\"pyglove dict\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003edict(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e{\\n  \u0026#x27;queries\u0026#x27;: [\\n    QueryInvocation(\\n      input=Ref(\\n        value = Template(\\n          template_str = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\\n          clean = True\\n        )\\n      ),\\n      schema=Schema(\\n        spec=Object(\\n          Answer\\n        )\\n      ),\\n      lm_response=AIMessage(\\n        text=\u0026#x27;```python\\\\nAnswer(\\\\n    step_by_step_thoughts=[\\\\n        \u0026quot;Janet\\\\\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\\\n        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\\\\n        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\\\\n        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\\\\n ...\u0026#x27;,\\n        sender=\u0026#x27;AI\u0026#x27;,\\n        metadata={},\\n        tags=[]\\n      ),\\n      lm=Ref(\\n        value = Gpt4oMini(\\n          sampling_options = LMSamplingOptions(\\n            temperature = None,\\n            max_tokens = None,\\n            n = 1,\\n            top_k = 40,\\n            top_p = None,\\n            stop = None,\\n            random_seed = None,\\n            logprobs = False,\\n            top_logprobs = None\\n          ),\\n          cache = ContextualAttribute(\\n            type = None,\\n            default = None\\n          ),\\n          max_concurrency = None,\\n          timeout = 120.0,\\n          max_attempts = 5,\\n          retry_interval = (5, 60),\\n          exponential_backoff = True,\\n          max_retry_interval = 300,\\n          debug = False,\\n          api_endpoint = \u0026#x27;https://api.openai.com/v1/chat/completions\u0026#x27;,\\n          model = \u0026#x27;gpt-4o-mini\u0026#x27;,\\n          headers = None,\\n          multimodal = True,\\n          api_key = None,\\n          organization = None,\\n          project = None\\n        )\\n      ),\\n      examples=[],\\n      usage_summary=UsageSummary(\\n        cached=AggregatedUsage(\\n          total=LMSamplingUsage(\\n            prompt_tokens=0,\\n            completion_tokens=0,\\n            total_tokens=0,\\n            num_requests=0,\\n            estimated_cost=0.0,\\n            retry_stats=RetryStats(\\n              num_occurences=0,\\n              total_wait_interval=0.0,\\n              total_call_interval=0.0,\\n              errors={}\\n            )\\n          ),\\n          breakdown={}\\n        ),\\n        uncached=AggregatedUsage(\\n          total=LMSamplingUsage(\\n            prompt_tokens=180,\\n            completion_tokens=136,\\n            total_tokens=316,\\n            num_requests=1,\\n            estimated_cost=0.00010859999999999998,\\n            retry_stats=RetryStats(\\n              num_occurences=0,\\n              total_wait_interval=0.0,\\n              total_call_interval=2.882235288619995,\\n              errors={}\\n            )\\n          ),\\n          breakdown={\\n            \u0026#x27;OpenAI(gpt-4o-mini)\u0026#x27;: LMSamplingUsage(\\n              prompt_tokens=180,\\n              completion_tokens=136,\\n              total_tokens=316,\\n              num_requests=1,\\n              estimated_cost=0.00010859999999999998,\\n              retry_stats=RetryStats(\\n                num_occurences=0,\\n                total_wait_interval=0.0,\\n                total_call_interval=2.882235288619995,\\n                errors={}\\n              )\\n            )\\n          }\\n        )\\n      ),\\n      start_time=1737569905.6691015,\\n      end_time=1737569908.6945176\\n    )\\n  ]\\n}\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value dict\"\u003e\u003cdetails open class=\"pyglove list\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003equeries\u003cspan class=\"tooltip\"\u003emetadata.queries\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003elist(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e[\\n  QueryInvocation(\\n    input=Ref(\\n      value = Template(\\n        template_str = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\\n        clean = True\\n      )\\n    ),\\n    schema=Schema(\\n      spec=Object(\\n        Answer\\n      )\\n    ),\\n    lm_response=AIMessage(\\n      text=\u0026#x27;```python\\\\nAnswer(\\\\n    step_by_step_thoughts=[\\\\n        \u0026quot;Janet\\\\\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\\\n        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\\\\n        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\\\\n        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\\\\n ...\u0026#x27;,\\n      sender=\u0026#x27;AI\u0026#x27;,\\n      metadata={},\\n      tags=[]\\n    ),\\n    lm=Ref(\\n      value = Gpt4oMini(\\n        sampling_options = LMSamplingOptions(\\n          temperature = None,\\n          max_tokens = None,\\n          n = 1,\\n          top_k = 40,\\n          top_p = None,\\n          stop = None,\\n          random_seed = None,\\n          logprobs = False,\\n          top_logprobs = None\\n        ),\\n        cache = ContextualAttribute(\\n          type = None,\\n          default = None\\n        ),\\n        max_concurrency = None,\\n        timeout = 120.0,\\n        max_attempts = 5,\\n        retry_interval = (5, 60),\\n        exponential_backoff = True,\\n        max_retry_interval = 300,\\n        debug = False,\\n        api_endpoint = \u0026#x27;https://api.openai.com/v1/chat/completions\u0026#x27;,\\n        model = \u0026#x27;gpt-4o-mini\u0026#x27;,\\n        headers = None,\\n        multimodal = True,\\n        api_key = None,\\n        organization = None,\\n        project = None\\n      )\\n    ),\\n    examples=[],\\n    usage_summary=UsageSummary(\\n      cached=AggregatedUsage(\\n        total=LMSamplingUsage(\\n          prompt_tokens=0,\\n          completion_tokens=0,\\n          total_tokens=0,\\n          num_requests=0,\\n          estimated_cost=0.0,\\n          retry_stats=RetryStats(\\n            num_occurences=0,\\n            total_wait_interval=0.0,\\n            total_call_interval=0.0,\\n            errors={}\\n          )\\n        ),\\n        breakdown={}\\n      ),\\n      uncached=AggregatedUsage(\\n        total=LMSamplingUsage(\\n          prompt_tokens=180,\\n          completion_tokens=136,\\n          total_tokens=316,\\n          num_requests=1,\\n          estimated_cost=0.00010859999999999998,\\n          retry_stats=RetryStats(\\n            num_occurences=0,\\n            total_wait_interval=0.0,\\n            total_call_interval=2.882235288619995,\\n            errors={}\\n          )\\n        ),\\n        breakdown={\\n          \u0026#x27;OpenAI(gpt-4o-mini)\u0026#x27;: LMSamplingUsage(\\n            prompt_tokens=180,\\n            completion_tokens=136,\\n            total_tokens=316,\\n            num_requests=1,\\n            estimated_cost=0.00010859999999999998,\\n            retry_stats=RetryStats(\\n              num_occurences=0,\\n              total_wait_interval=0.0,\\n              total_call_interval=2.882235288619995,\\n              errors={}\\n            )\\n          )\\n        }\\n      )\\n    ),\\n    start_time=1737569905.6691015,\\n    end_time=1737569908.6945176\\n  )\\n]\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value list\"\u003e\u003ctable\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e0\u003c/span\u003e\u003cspan class=\"tooltip\"\u003emetadata.queries[0]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cdetails open class=\"pyglove query-invocation\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003e\u003cdiv class=\"query-invocation-title\"\u003e\u003cspan class=\"label query-invocation-type-name\"\u003elf.query\u003c/span\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label query-invocation-lm badge\"\u003elm=OpenAI(gpt-4o-mini)\u003c/span\u003e\u003cspan class=\"tooltip\"\u003eGpt4oMini()\u003c/span\u003e\u003c/div\u003e\u003cspan class=\"label query-invocation-time badge\"\u003e3 seconds\u003c/span\u003e\u003cdiv class=\"label-container\"\u003e\u003cspan class=\"label usage-summary badge\" id=\"control-138085542613200\"\u003e0.000\u003c/span\u003e\u003cspan class=\"tooltip\" id=\"control-138085545041744\"\u003eUsageSummary(\\n  cached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 0,\\n      completion_tokens = 0,\\n      total_tokens = 0,\\n      num_requests = 0,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 0.000,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {}\\n  ),\\n  uncached = AggregatedUsage(\\n    total = LMSamplingUsage(\\n      prompt_tokens = 180,\\n      completion_tokens = 136,\\n      total_tokens = 316,\\n      num_requests = 1,\\n      estimated_cost = 0.000,\\n      retry_stats = RetryStats(\\n        num_occurences = 0,\\n        total_wait_interval = 0.000,\\n        total_call_interval = 2.882,\\n        errors = {}\\n      )\\n    ),\\n    breakdown = {\\n      OpenAI(gpt-4o-mini) = LMSamplingUsage(\\n        prompt_tokens = 180,\\n        completion_tokens = 136,\\n        total_tokens = 316,\\n        num_requests = 1,\\n        estimated_cost = 0.000,\\n        retry_stats = RetryStats(\\n          num_occurences = 0,\\n          total_wait_interval = 0.000,\\n          total_call_interval = 2.882,\\n          errors = {}\\n        )\\n      )\\n    }\\n  )\\n)\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/summary\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085516702992-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085516702992\\', \\'control-138085516702992-0\\')\"\u003e\u003cspan class=\"label\"\u003einput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, \\'control-138085516702992\\', \\'control-138085516702992-1\\')\"\u003e\u003cspan class=\"label\"\u003eoutput\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085516702992\\', \\'control-138085516702992-2\\')\"\u003e\u003cspan class=\"label\"\u003eschema\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085516702992\\', \\'control-138085516702992-3\\')\"\u003e\u003cspan class=\"label\"\u003elm_request\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085516702992\\', \\'control-138085516702992-4\\')\"\u003e\u003cspan class=\"label\"\u003elm_response\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085516702992-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-0\"\u003e\u003cdetails open class=\"pyglove template lf-template\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title lf-template\"\u003eTemplate(...)\u003c/div\u003e\u003cspan class=\"tooltip lf-template\"\u003eTemplate(\\n  template_str=\u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day...\u0026quot;,\\n  clean=True\\n)\u003c/span\u003e\u003c/summary\u003e\u003ctable class=\"tab-control\"\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-button-group top\" id=\"control-138085516558480-button-group\"\u003e\u003cbutton class=\"tab-button\" onclick=\"openTab(event, \\'control-138085516558480\\', \\'control-138085516558480-0\\')\"\u003e\u003cspan class=\"label\"\u003etemplate_str\u003c/span\u003e\u003c/button\u003e\u003cbutton class=\"tab-button selected\" onclick=\"openTab(event, \\'control-138085516558480\\', \\'control-138085516558480-1\\')\"\u003e\u003cspan class=\"label\"\u003evariables\u003c/span\u003e\u003c/button\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cdiv class=\"tab-content-group top\" id=\"control-138085516558480-content-group\"\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516558480-0\"\u003e\u003cdiv class=\"template-str\"\u003e\u003cspan\u003eJanet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\\' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\\' market?\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085516558480-1\"\u003e\u003cdiv class=\"complex-value template\"\u003e\u003cspan class=\"empty-container\"\u003e\u003c/span\u003e\u003c/div\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content selected\" id=\"control-138085516702992-1\"\u003e\u003cdetails open class=\"pyglove answer\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eAnswer(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003eAnswer(\\n  step_by_step_thoughts=[\\n    \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n    \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\\n    \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\\n    \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\\n    \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\\n    \u0026#x27;She sells each egg for $2.\u0026#x27;,\\n    \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\\n  ],\\n  final_answer=18\\n)\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value answer\"\u003e\u003cdetails open class=\"pyglove list\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003estep_by_step_thoughts\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eList(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e[\\n  \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n  \u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;,\\n  \u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;,\\n  \u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;,\\n  \u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;,\\n  \u0026#x27;She sells each egg for $2.\u0026#x27;,\\n  \u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\\n]\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex-value list\"\u003e\u003ctable\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e0\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[0]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e1\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[1]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She eats 3 eggs for breakfast.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e2\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[2]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She uses 4 eggs for baking muffins.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e3\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[3]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e4\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[4]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e5\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[5]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;She sells each egg for $2.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e\u003cspan class=\"object-key int\"\u003e6\u003c/span\u003e\u003cspan class=\"tooltip\"\u003estep_by_step_thoughts[6]\u003c/span\u003e\u003c/td\u003e\u003ctd\u003e\u003cspan class=\"simple-value str\"\u003e\u0026#x27;Total money made = 9 eggs * $2/egg = $18.\u0026#x27;\u003c/span\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/details\u003e\u003cdetails open class=\"pyglove int\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-name\"\u003efinal_answer\u003cspan class=\"tooltip\"\u003efinal_answer\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"summary-title\"\u003eint\u003c/div\u003e\u003cspan class=\"tooltip\"\u003e18\u003c/span\u003e\u003c/summary\u003e\u003cspan class=\"simple-value int\"\u003e18\u003c/span\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-2\"\u003e\u003cdetails open class=\"pyglove schema\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title\"\u003eSchema(...)\u003c/div\u003e\u003cspan class=\"tooltip\"\u003eSchema(\\n  spec=Object(\\n    Answer\\n  )\\n)\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"lf-schema-definition\"\u003eAnswer\\n\\n```python\\nclass Answer:\\n  step_by_step_thoughts: list[str]\\n  final_answer: int\\n```\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-3\"\u003e\u003cdetails open class=\"pyglove user-message lf-message\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title lf-message\"\u003eUserMessage(...)\u003c/div\u003e\u003cspan class=\"tooltip lf-message\"\u003eUserMessage(\\n  text=\u0026#x27;Please respond to the last INPUT_OBJECT with OUTPUT_OBJECT according to OUTPUT_TYPE.\\\\n\\\\nINPUT_OBJECT:\\\\n  1 + 1 =\\\\n\\\\nOUTPUT_TYPE:\\\\n  Answer\\\\n\\\\n  ```python\\\\n  class Answer:\\\\n    final_answer: int\\\\n  ```\\\\n\\\\nOUTPUT_OBJECT:\\\\n  ```python\\\\n  Answer(\\\\n    final_answer=2\\\\n  )\\\\n  ```...\u0026#x27;,\\n  sender=\u0026#x27;User\u0026#x27;,\\n  metadata={\\n    \u0026#x27;mapping_template\u0026#x27;: Ref(\\n      value = Template(\\n        template_str = \u0026#x27;{%- if example.context -%}\\\\n{{ context_title}}:\\\\n{{ example.context | indent(2, True)}}\\\\n\\\\n{% endif -%}\\\\n\\\\n{{ input_title }}:\\\\n{{ example.input_repr(protocol, compact=False) | indent(2, True) }}\\\\n\\\\n{% if example.schema -%}\\\\n{{ schema_title }}:\\\\n{{ example.schema_repr(protocol) | indent(2, True) }}\\\\n\\\\n{% endif -%}\\\\n\\\\n{{ output_title }}:\\\\n{%- if example.has_output %}\\\\n{{ example.output_repr(protocol, compact=False) | indent(2, True) }}\\\\n{% endif -%}\u0026#x27;,\\n        clean = True,\\n        context_title = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        input_title = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        output_title = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        schema_title = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        protocol = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        example = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        )\\n      )\\n    ),\\n    \u0026#x27;examples\u0026#x27;: None,\\n    \u0026#x27;preamble\u0026#x27;: Ref(\\n      value = LangFunc(\\n        template_str = \u0026#x27;Please respond to the last {{ input_title }} with {{ output_title }} according to {{ schema_title }}.\\\\n\\\\n{{ input_title }}:\\\\n  1 + 1 =\\\\n\\\\n{{ schema_title }}:\\\\n  Answer\\\\n\\\\n  ```python\\\\n  class Answer:\\\\n    final_answer: int\\\\n  ```\\\\n\\\\n{{ output_title }}:\\\\n  ```python\\\\n  Answer(\\\\n    final_answer=2\\\\n  )\\\\n  ```\u0026#x27;,\\n        clean = True,\\n        lm = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        input_title = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        output_title = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        ),\\n        schema_title = ContextualAttribute(\\n          type = None,\\n          default = (MISSING_VALUE)\\n        )\\n      )\\n    ),\\n    \u0026#x27;mapping_request\u0026#x27;: Ref(\\n      value = MappingExample(\\n        input = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\\n        output = MISSING(Any),\\n        schema = Ref(\\n          value = Schema(\\n            spec = Object(\\n              Answer\\n            )\\n          )\\n        ),\\n        context = None,\\n        metadata = {}\\n      )\\n    ),\\n    \u0026#x27;input\u0026#x27;: Ref(\\n      value = UserMessage(\\n        text = \u0026quot;Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\u0026quot;,\\n        sender = \u0026#x27;User\u0026#x27;,\\n        metadata = {},\\n        tags = [\\n          0 : \u0026#x27;rendered\u0026#x27;\\n        ]\\n      )\\n    ),\\n    \u0026#x27;schema\u0026#x27;: Ref(\\n      value = Schema(\\n        spec = Object(\\n          Answer\\n        )\\n      )\\n    )\\n  },\\n  tags=[\\n    \u0026#x27;rendered\u0026#x27;\\n  ]\\n)\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex_value\"\u003e\u003cdiv class=\"message-tags\"\u003e\u003cspan\u003erendered\u003c/span\u003e\u003c/div\u003e\u003cdiv class=\"message-text\"\u003ePlease respond to the last INPUT_OBJECT with OUTPUT_OBJECT according to OUTPUT_TYPE.\\n\\nINPUT_OBJECT:\\n  1 + 1 =\\n\\nOUTPUT_TYPE:\\n  Answer\\n\\n  ```python\\n  class Answer:\\n    final_answer: int\\n  ```\\n\\nOUTPUT_OBJECT:\\n  ```python\\n  Answer(\\n    final_answer=2\\n  )\\n  ```\\n\\nINPUT_OBJECT:\\n  Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\u0026#x27; market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\u0026#x27; market?\\n\\nOUTPUT_TYPE:\\n  Answer\\n\\n  ```python\\n  class Answer:\\n    step_by_step_thoughts: list[str]\\n    final_answer: int\\n  ```\\n\\nOUTPUT_OBJECT:\u003c/div\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003cdiv class=\"tab-content\" id=\"control-138085516702992-4\"\u003e\u003cdetails open class=\"pyglove ai-message lf-message\"\u003e\u003csummary\u003e\u003cdiv class=\"summary-title lf-message\"\u003eAIMessage(...)\u003c/div\u003e\u003cspan class=\"tooltip lf-message\"\u003eAIMessage(\\n  text=\u0026#x27;```python\\\\nAnswer(\\\\n    step_by_step_thoughts=[\\\\n        \u0026quot;Janet\\\\\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\\\n        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\\\\n        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\\\\n        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\\\\n ...\u0026#x27;,\\n  sender=\u0026#x27;AI\u0026#x27;,\\n  metadata={},\\n  tags=[]\\n)\u003c/span\u003e\u003c/summary\u003e\u003cdiv class=\"complex_value\"\u003e\u003cdiv class=\"message-tags\"\u003e\u003c/div\u003e\u003cdiv class=\"message-text\"\u003e```python\\nAnswer(\\n    step_by_step_thoughts=[\\n        \u0026quot;Janet\u0026#x27;s ducks lay 16 eggs per day.\u0026quot;,\\n        \u0026quot;She eats 3 eggs for breakfast.\u0026quot;,\\n        \u0026quot;She uses 4 eggs for baking muffins.\u0026quot;,\\n        \u0026quot;Total eggs used per day = 3 (breakfast) + 4 (baking) = 7 eggs.\u0026quot;,\\n        \u0026quot;Eggs left to sell = 16 (total) - 7 (used) = 9 eggs.\u0026quot;,\\n        \u0026quot;She sells each egg for $2.\u0026quot;,\\n        \u0026quot;Total money made = 9 eggs * $2/egg = $18.\u0026quot;\\n    ],\\n    final_answer=18\\n)\\n```\u003c/div\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/details\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003c/details\u003e\u003c/div\u003e\u003c/div\u003e\u003c/td\u003e\u003c/tr\u003e\u003c/table\u003e\u003c/div\u003e', style_files=StyleFiles(), styles=Styles(parts={'\\n        .label {\\n          display: inline-block;\\n          color: inherit;\\n          padding: 5px;\\n        }\\n        .label-container {\\n            display: inline-block;\\n        }\\n        ': 18, '\\n        span.tooltip {\\n          visibility: hidden;\\n          white-space: pre-wrap;\\n          font-weight: normal;\\n          background-color: #484848;\\n          color: #fff;\\n          padding: 10px;\\n          border-radius: 6px;\\n          position: absolute;\\n          z-index: 1;\\n        }\\n        span.tooltip:hover {\\n          visibility: visible;\\n        }\\n        .tooltip.html-content {\\n          white-space: inherit;\\n          background-color: white;\\n          color: inherit;\\n          box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px;\\n        }\\n        ': 3, '\\n        .label:hover + .tooltip {\\n          visibility: visible;\\n        }\\n        ': 3, '\\n        .badge {\\n          background-color: #EEE;\\n          border-radius: 5px;\\n          color: #777;\\n        }\\n        ': 5, \"\\n        .usage-summary.label {\\n            display: inline-flex;\\n            border-radius: 5px;\\n            padding: 5px;\\n            background-color: #f1f1f1;\\n            color: #CCC;\\n        }\\n        .usage-summary.label::before {\\n            content: '$';\\n        }\\n        \": 2, '\\n        .label-group {\\n          display: inline-flex;\\n          overflow: hidden;\\n          border-radius: 5px;\\n          border: 1px solid #DDD;\\n          padding: 0px;\\n          margin: 5px\\n        }\\n        .label-group .group-name {\\n          padding: 5px;\\n          color: white;\\n          background-color: dodgerblue;\\n        }\\n        .label-group .group-name \u003e .text {\\n          color: white;\\n        }\\n        .label-group .group-value {\\n        }\\n        .label-group .badge {\\n          border-radius: 0px;\\n        }\\n        ': 1, '\\n        /* Tooltip styles. */\\n        span.tooltip {\\n          visibility: hidden;\\n          white-space: pre-wrap;\\n          font-weight: normal;\\n          background-color: #484848;\\n          color: #fff;\\n          padding: 10px;\\n          border-radius: 6px;\\n          position: absolute;\\n          z-index: 1;\\n        }\\n        span.tooltip:hover {\\n          visibility: visible;\\n        }\\n        ': 37, '\\n        /* Summary styles. */\\n        details.pyglove summary {\\n          font-weight: bold;\\n          margin: -0.5em -0.5em 0;\\n          padding: 0.5em;\\n        }\\n        .summary-name {\\n          display: inline;\\n          padding: 3px 5px 3px 5px;\\n          margin: 0 5px;\\n          border-radius: 3px;\\n        }\\n        .summary-title {\\n          display: inline;\\n        }\\n        .summary-name + div.summary-title {\\n          display: inline;\\n          color: #aaa;\\n        }\\n        .summary-title:hover + span.tooltip {\\n          visibility: visible;\\n        }\\n        .summary-name:hover \u003e span.tooltip {\\n          visibility: visible;\\n          background-color: darkblue;\\n        }\\n        ': 16, '\\n        /* Simple value styles. */\\n        .simple-value {\\n          color: blue;\\n          display: inline-block;\\n          white-space: pre-wrap;\\n          padding: 0.2em;\\n          margin-top: 0.15em;\\n        }\\n        .simple-value.str {\\n          color: darkred;\\n          font-style: italic;\\n        }\\n        .simple-value.int, .simple-value.float {\\n          color: darkblue;\\n        }\\n        ': 18, '\\n        /* Value details styles. */\\n        details.pyglove {\\n          border: 1px solid #aaa;\\n          border-radius: 4px;\\n          padding: 0.5em 0.5em 0;\\n          margin: 0.25em 0;\\n        }\\n        details.pyglove[open] {\\n          padding: 0.5em 0.5em 0.5em;\\n        }\\n        .highlight {\\n          background-color: Mark;\\n        }\\n        .lowlight {\\n          opacity: 0.2;\\n        }\\n        ': 16, \"\\n        /* Complex value styles. */\\n        span.empty-container::before {\\n            content: '(empty)';\\n            font-style: italic;\\n            margin-left: 0.5em;\\n            color: #aaa;\\n        }\\n        \": 8, \"\\n        /* Object key styles. */\\n        .object-key {\\n          margin: 0.15em 0.3em 0.15em 0;\\n          display: block;\\n        }\\n        .object-key:hover + .tooltip {\\n          visibility: visible;\\n          background-color: darkblue;\\n        }\\n        .object-key.str {\\n          color: gray;\\n          border: 1px solid lightgray;\\n          background-color: ButtonFace;\\n          border-radius: 0.2em;\\n          padding: 0.3em;\\n        }\\n        .object-key.int::before{\\n          content: '[';\\n        }\\n        .object-key.int::after{\\n          content: ']';\\n        }\\n        .object-key.int{\\n          border: 0;\\n          color: lightgray;\\n          background-color: transparent;\\n          border-radius: 0;\\n          padding: 0;\\n        }\\n        \": 15, '\\n        .tab-control {\\n          border-spacing: 0px;\\n          border-collapse: collapse;\\n          margin-top: 10px;\\n        }\\n        .tab-control td {\\n          padding: 0px;\\n          margin: 0px;\\n          vertical-align: top;\\n        }\\n        .top.tab-button-group {\\n          border-left: 1px solid #DDD;\\n          border-top: 1px solid #DDD;\\n          border-right: 1px solid #DDD;\\n          border-radius: 5px 5px 0px 0px;\\n          padding: 0px 5px 0px 0px;\\n          margin-bottom: -2px;\\n        }\\n        .tab-button {\\n          background-color: transparent;\\n          border-radius: 5px;\\n          border: 0px;\\n          font-weight: bold;\\n          color: gray;\\n          outline: none;\\n          cursor: pointer;\\n          transition: 0.3s;\\n        }\\n        .tab-button:hover {\\n          background-color: #fff1dd;\\n        }\\n        .tab-button.selected {\\n          background-color: #f0ecf9;\\n          color: black;\\n        }\\n        .top.tab-content-group {\\n          border-left: 1px solid #DDD;\\n          border-right: 1px solid #DDD;\\n          border-bottom: 1px solid #DDD;\\n          border-radius: 0px 0px 5px 5px;\\n          margin: 0px;\\n          padding: 5px;\\n          height: 100%;\\n        }\\n        .top \u003e .tab-button {\\n          margin: 5px 0px 5px 5px;\\n        }\\n        .tab-content {\\n          display: none;\\n        }\\n        .tab-content.selected {\\n          display: block;\\n        }\\n        .left.tab-button-group {\\n          display: inline-flex;\\n          flex-direction: column;\\n          border: 1px solid #DDD;\\n          border-radius: 5px;\\n          margin-right: 5px;\\n          padding: 0px 0px 5px 0px;\\n        }\\n        .left.tab-content-group {\\n          border: 0px\\n          margin: 0px;\\n          padding: 0px;\\n          height: 100%;\\n        }\\n        .left \u003e .tab-button {\\n          text-align: left;\\n          margin: 5px 5px 0px 5px;\\n        }\\n        ': 3, '\\n        /* Langfun Template styles. */\\n        .template-str {\\n            padding: 10px;\\n            margin: 10px 5px 10px 5px;\\n            font-style: italic;\\n            font-size: 1.1em;\\n            white-space: pre-wrap;\\n            border: 1px solid #EEE;\\n            border-radius: 5px;\\n            background-color: #EEE;\\n            color: #cc2986;\\n        }\\n        ': 1, '\\n        .lf-schema-definition {\\n            color: blue;\\n            margin: 5px;\\n            white-space: pre-wrap;\\n        }\\n        ': 1, '\\n        /* Langfun Message styles.*/\\n        [class^=\"message-\"] \u003e details {\\n            margin: 0px 0px 5px 0px;\\n            border: 1px solid #EEE;\\n        }\\n        .lf-message.summary-title::after {\\n            content: \\' 💬\\';\\n        }\\n        details.pyglove.ai-message {\\n            border: 1px solid blue;\\n            color: blue;\\n        }\\n        details.pyglove.user-message {\\n            border: 1px solid green;\\n            color: green;\\n        }\\n        .message-tags {\\n            margin: 5px 0px 5px 0px;\\n            font-size: .8em;\\n        }\\n        .message-tags \u003e span {\\n            border-radius: 5px;\\n            background-color: #CCC;\\n            padding: 3px;\\n            margin: 0px 2px 0px 2px;\\n            color: white;\\n        }\\n        .message-text {\\n            padding: 20px;\\n            margin: 10px 5px 10px 5px;\\n            font-style: italic;\\n            white-space: pre-wrap;\\n            border: 1px solid #EEE;\\n            border-radius: 5px;\\n            background-color: #EEE;\\n        }\\n        .modality-in-text {\\n            display: inline-block;\\n        }\\n        .modality-in-text \u003e details.pyglove {\\n            display: inline-block;\\n            font-size: 0.8em;\\n            border: 0;\\n            background-color: #A6F1A6;\\n            margin: 0px 5px 0px 5px;\\n        }\\n        .message-result {\\n            color: dodgerblue;\\n        }\\n        .message-usage {\\n            color: orange;\\n        }\\n        .message-usage .object-key.str {\\n            border: 1px solid orange;\\n            background-color: orange;\\n            color: white;\\n        }\\n        ': 2, '\\n        .query-invocation-title {\\n          display: inline-block;\\n          font-weight: normal;\\n        }\\n        .query-invocation-type-name {\\n          color: #888;\\n        }\\n        .query-invocation-lm.badge {\\n          margin-left: 5px;\\n          margin-right: 5px;\\n          color: white;\\n          background-color: mediumslateblue;\\n        }\\n        .query-invocation-time.badge {\\n          margin-left: 5px;\\n          border-radius: 0px;\\n          font-weight: bold;\\n          background-color: aliceblue;\\n        }\\n        .query-invocation-title .usage-summary.label {\\n          border-radius: 0px;\\n          color: #AAA;\\n        }\\n        ': 1, '\\n        .example-container {\\n          display: block;\\n          padding: 10px;\\n        }\\n        .example-id {\\n          font-weight: bold;\\n          font-size: 40px;\\n          margin: 0 10px;\\n          vertical-align: middle;\\n        }\\n        a.previous, a.next {\\n          text-decoration: none;\\n          vertical-align: middle;\\n          display: inline-block;\\n          padding: 8px 8px;\\n          color: #DDD;\\n        }\\n        a.previous:hover, a.next:hover {\\n          background-color: #ddd;\\n          color: black;\\n        }\\n        /* Badge styles. */\\n        .eval-example .badge.match {\\n          color: green;\\n          background-color: #dcefbe;\\n        }\\n        .eval-example .badge.error {\\n          color: red;\\n          background-color: #fdcccc;\\n        }\\n        .eval-example .badge.mismatch {\\n          color: orange;\\n          background-color: #ffefc4;\\n        }\\n        .eval-example .badge.score {\\n          color: blue;\\n          background-color: #c4dced;\\n        }\\n        ': 1}), script_files=ScriptFiles(), scripts=Scripts(parts={\"\\n        function openTab(event, controlId, tabId) {\\n          const tabButtons = document.querySelectorAll('#' + controlId + '-button-group \u003e .tab-button');\\n          for (let i = 0; i \u003c tabButtons.length; i++) {\\n            tabButtons[i].classList.remove('selected');\\n          }\\n          const tabContents = document.querySelectorAll('#' + controlId + '-content-group \u003e .tab-content');\\n          for (let i = 0; i \u003c tabContents.length; i++) {\\n            tabContents[i].classList.remove('selected')\\n          }\\n          const tabButton = event.currentTarget;\\n          tabButton.classList.add('selected');\\n          document.getElementById(tabId).classList.add('selected');\\n        }\\n        \": 3}))"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "example_id = 1  # @param int\n",
        "pg.view(benchmark.leaf_nodes[0].state.evaluated_examples[example_id])"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [
        {
          "file_id": "1wzjNOvf-Bfq7Geyi7eL-EXy6H8GhQDxL",
          "timestamp": 1737580416747
        }
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
